tessl/npm-ipld--car

Content Addressable aRchive format reader and writer for IPLD data structures.

—

Pending

Quality

Pending

Does it follow best practices?

Impact

Pending

No eval scenarios have been run

Securityby

Pending

The risk profile of this skill

Overview

Eval results

Files

Buffer Writing (Node.js)

Name: tessl/npm-ipld--car
Author: tessl

Synchronous buffer-based CAR writing for performance-critical scenarios where the final CAR size is known in advance. Provides direct buffer manipulation with precise memory control and no async overhead. This functionality is only available in Node.js.

Capabilities

CarBufferWriter Class

Synchronous CAR writer that writes directly to a pre-allocated buffer.

/**
 * Synchronous CAR writer for pre-allocated buffers
 * Provides precise memory control and no async overhead
 * Suitable when final CAR size is known in advance
 * Node.js only - not available in browser environments
 */
interface CarBufferWriter {
  /** Buffer being written to */
  readonly bytes: Uint8Array;
  /** Current root CIDs */
  readonly roots: CID[];
  
  /** Add a root CID to the header, optionally resizing header space */
  addRoot(root: CID, options?: { resize?: boolean }): CarBufferWriter;
  
  /** Write a block to the buffer, throws if insufficient space */
  write(block: Block): CarBufferWriter;
  
  /** Finalize CAR and return final byte array, optionally resizing */
  close(options?: { resize?: boolean }): Uint8Array;
}

/**
 * Options for creating a buffer writer
 */
interface CarBufferWriterOptions {
  /** Initial root CIDs (default: []) */
  roots?: CID[];
  /** Byte offset in buffer to start writing (default: 0) */
  byteOffset?: number;
  /** Number of bytes to use from buffer (default: buffer.byteLength) */
  byteLength?: number;
  /** Reserved space for header (default: calculated from roots) */
  headerSize?: number;
}

Creating Buffer Writers

Create buffer writers with precise memory allocation.

/**
 * Create buffer writer from ArrayBuffer with optional configuration
 * @param buffer - ArrayBuffer to write CAR data into
 * @param options - Configuration options
 * @returns CarBufferWriter instance
 */
function createWriter(buffer: ArrayBuffer, options?: CarBufferWriterOptions): CarBufferWriter;

Usage Examples:

import * as CarBufferWriter from "@ipld/car/buffer-writer";

// Create buffer with estimated size
const estimatedSize = 1024 * 1024; // 1MB
const buffer = new ArrayBuffer(estimatedSize);

// Create writer with known roots
const writer = CarBufferWriter.createWriter(buffer, {
  roots: [rootCid1, rootCid2],
  headerSize: CarBufferWriter.headerLength({ roots: [rootCid1, rootCid2] })
});

// Write blocks
writer
  .write({ cid: blockCid1, bytes: blockData1 })
  .write({ cid: blockCid2, bytes: blockData2 })
  .write({ cid: blockCid3, bytes: blockData3 });

// Finalize and get result
const carBytes = writer.close();
console.log(`Created CAR: ${carBytes.length} bytes`);

Size Calculation Functions

Calculate buffer sizes and header requirements.

/**
 * Calculate bytes needed for storing a block in CAR format
 * @param block - Block to calculate size for
 * @returns Number of bytes needed
 */
function blockLength(block: Block): number;

/**
 * Calculate header length for given roots
 * @param options - Object containing roots array
 * @returns Header length in bytes
 */
function headerLength(options: { roots: CID[] }): number;

/**
 * Calculate header length from root CID byte lengths
 * @param rootLengths - Array of root CID byte lengths
 * @returns Header length in bytes
 */
function calculateHeaderLength(rootLengths: number[]): number;

/**
 * Estimate header length for a given number of roots
 * @param rootCount - Number of root CIDs
 * @param rootByteLength - Expected bytes per root CID (default: 36)
 * @returns Estimated header length in bytes
 */
function estimateHeaderLength(rootCount: number, rootByteLength?: number): number;

Usage Examples:

import * as CarBufferWriter from "@ipld/car/buffer-writer";

// Calculate exact buffer size needed
const blocks = [
  { cid: cid1, bytes: data1 },
  { cid: cid2, bytes: data2 },
  { cid: cid3, bytes: data3 }
];

const roots = [cid1];

// Calculate total size needed
const headerSize = CarBufferWriter.headerLength({ roots });
const blockSizes = blocks.reduce(
  (total, block) => total + CarBufferWriter.blockLength(block), 
  0
);
const totalSize = headerSize + blockSizes;

// Create perfectly sized buffer
const buffer = new ArrayBuffer(totalSize);
const writer = CarBufferWriter.createWriter(buffer, { 
  roots,
  headerSize 
});

// Write all blocks
for (const block of blocks) {
  writer.write(block);
}

const carBytes = writer.close();
console.log(`Perfect fit: ${carBytes.length} === ${totalSize}`);

Dynamic Header Resizing

Handle scenarios where root count changes during writing.

import * as CarBufferWriter from "@ipld/car/buffer-writer";

// Start with estimated header size
const buffer = new ArrayBuffer(1024 * 1024);
const estimatedHeaderSize = CarBufferWriter.estimateHeaderLength(5); // Expect ~5 roots

const writer = CarBufferWriter.createWriter(buffer, {
  headerSize: estimatedHeaderSize
});

// Add roots dynamically with resize option
try {
  writer.addRoot(root1);
  writer.addRoot(root2);
  writer.addRoot(root3);
  
  // This might exceed estimated header size
  writer.addRoot(root4, { resize: true }); // Automatically resize header
  writer.addRoot(root5, { resize: true });
  
} catch (error) {
  if (error instanceof RangeError && error.message.includes('resize')) {
    console.log('Use { resize: true } to expand header');
  }
}

// Write blocks and close with resize
writer
  .write(block1)
  .write(block2);

const carBytes = writer.close({ resize: true }); // Resize to actual header size

Advanced Buffer Management

Manage buffer allocation and reuse patterns.

import * as CarBufferWriter from "@ipld/car/buffer-writer";

// Pattern 1: Buffer pools for repeated CAR creation
class CarBufferPool {
  constructor(bufferSize = 1024 * 1024) {
    this.bufferSize = bufferSize;
    this.availableBuffers = [];
  }
  
  getBuffer() {
    return this.availableBuffers.pop() || new ArrayBuffer(this.bufferSize);
  }
  
  returnBuffer(buffer) {
    if (buffer.byteLength === this.bufferSize) {
      this.availableBuffers.push(buffer);
    }
  }
  
  async createCar(roots, blocks) {
    const buffer = this.getBuffer();
    
    try {
      const writer = CarBufferWriter.createWriter(buffer, { roots });
      
      for (const block of blocks) {
        writer.write(block);
      }
      
      return writer.close({ resize: true });
    } finally {
      this.returnBuffer(buffer);
    }
  }
}

// Pattern 2: Precise allocation for known data sets
function createOptimalCar(roots, blocks) {
  // Calculate exact size needed
  const headerSize = CarBufferWriter.headerLength({ roots });
  const dataSize = blocks.reduce(
    (total, block) => total + CarBufferWriter.blockLength(block),
    0
  );
  
  // Create exactly sized buffer
  const buffer = new ArrayBuffer(headerSize + dataSize);
  const writer = CarBufferWriter.createWriter(buffer, { 
    roots, 
    headerSize 
  });
  
  // Write all data
  blocks.forEach(block => writer.write(block));
  
  // No resize needed - should be perfect fit
  return writer.close();
}

Error Handling

Handle buffer overflow and sizing errors.

import * as CarBufferWriter from "@ipld/car/buffer-writer";

// Buffer capacity errors
const smallBuffer = new ArrayBuffer(1024); // Too small
const writer = CarBufferWriter.createWriter(smallBuffer);

try {
  writer.write(largeBlock); // Might exceed buffer capacity
} catch (error) {
  if (error instanceof RangeError && error.message.includes('capacity')) {
    console.log('Buffer too small for block');
    
    // Calculate needed size and recreate
    const neededSize = CarBufferWriter.blockLength(largeBlock);
    console.log(`Need at least ${neededSize} bytes`);
  }
}

// Header sizing errors
try {
  writer.addRoot(newRoot); // Might exceed header space
} catch (error) {
  if (error.message.includes('resize')) {
    console.log('Header space exceeded - use resize option');
    writer.addRoot(newRoot, { resize: true });
  }
}

// Close sizing errors
try {
  const carBytes = writer.close();
} catch (error) {
  if (error instanceof RangeError && error.message.includes('overestimated')) {
    console.log('Header was overestimated - use resize option');
    const carBytes = writer.close({ resize: true });
  }
}

Performance Optimization

Optimize for different performance scenarios.

import * as CarBufferWriter from "@ipld/car/buffer-writer";

// High-throughput CAR creation
class HighThroughputCarWriter {
  constructor() {
    // Pre-calculate common header sizes
    this.headerSizeCache = new Map();
    
    // Reuse buffers for similar-sized outputs
    this.bufferPool = new Map(); // size -> [buffers]
  }
  
  precalculateHeaderSize(rootCount) {
    if (!this.headerSizeCache.has(rootCount)) {
      const size = CarBufferWriter.estimateHeaderLength(rootCount);
      this.headerSizeCache.set(rootCount, size);
    }
    return this.headerSizeCache.get(rootCount);
  }
  
  getPooledBuffer(size) {
    const roundedSize = Math.ceil(size / 1024) * 1024; // Round to KB
    const pool = this.bufferPool.get(roundedSize) || [];
    
    if (pool.length > 0) {
      return pool.pop();
    }
    
    return new ArrayBuffer(roundedSize);
  }
  
  returnBuffer(buffer) {
    const size = buffer.byteLength;
    const pool = this.bufferPool.get(size) || [];
    
    if (pool.length < 10) { // Limit pool size
      pool.push(buffer);
      this.bufferPool.set(size, pool);
    }
  }
  
  createCar(roots, blocks) {
    // Fast path calculations
    const headerSize = this.precalculateHeaderSize(roots.length);
    const dataSize = blocks.reduce(
      (total, block) => total + CarBufferWriter.blockLength(block),
      0
    );
    
    const buffer = this.getPooledBuffer(headerSize + dataSize);
    
    try {
      const writer = CarBufferWriter.createWriter(buffer, { 
        roots, 
        headerSize 
      });
      
      // Batch write blocks
      blocks.forEach(block => writer.write(block));
      
      return writer.close({ resize: true });
    } finally {
      this.returnBuffer(buffer);
    }
  }
}