or run

npx @tessl/cli init
Log in

Version

Tile

Overview

Evals

Files

docs

files.mdindex.mdprotocols.mdstreams.mdtypes.mdutils.md
tile.json

tessl/npm-avro-js

Pure JavaScript implementation of Apache Avro specification for fast binary serialization with schema evolution support

Workspace
tessl
Visibility
Public
Created
Last updated
Describes
npmpkg:npm/avro-js@1.12.x

To install, run

npx @tessl/cli install tessl/npm-avro-js@1.12.0

index.mddocs/

Avro-js

Avro-js is a pure JavaScript implementation of the Apache Avro specification that provides fast binary serialization and deserialization capabilities. It offers twice the speed of JSON with much smaller encodings, supports the full Avro specification including recursive schemas, sort order, schema evolution, and serialization of arbitrary JavaScript objects via logical types.

Package Information

  • Package Name: avro-js
  • Package Type: npm
  • Language: JavaScript
  • Installation: npm install avro-js

Core Imports

const avro = require('avro-js');
const { Type, Protocol, parse } = require('avro-js');

For ES modules:

import avro from 'avro-js';
import { Type, Protocol, parse } from 'avro-js';

For protocol-specific functionality:

const protocols = require('avro-js/lib/protocols');
const { createProtocol } = protocols;

Basic Usage

const avro = require('avro-js');

// Parse a schema and create a type
const schema = {
  type: 'record',
  name: 'User',
  fields: [
    { name: 'id', type: 'int' },
    { name: 'name', type: 'string' },
    { name: 'email', type: ['null', 'string'], default: null }
  ]
};

const userType = avro.parse(schema);

// Serialize data
const user = { id: 1, name: 'Alice', email: 'alice@example.com' };
const buffer = userType.toBuffer(user);

// Deserialize data
const deserializedUser = userType.fromBuffer(buffer);
console.log(deserializedUser); // { id: 1, name: 'Alice', email: 'alice@example.com' }

Architecture

Avro-js is built around several key components:

  • Type System: Complete implementation of Avro types including primitives, complex types, and logical types
  • Schema Parsing: Flexible schema parsing supporting JSON schemas, file paths, and protocol definitions
  • File Processing: Avro container file reading/writing with compression support
  • Protocol System: RPC/IPC implementation with emitters and listeners for client-server communication
  • Stream API: Node.js streams for efficient processing of large datasets
  • Browser Support: Browserify-compatible build for client-side usage

Capabilities

Schema Parsing and Types

Core schema parsing functionality that converts Avro schemas into executable Type instances. Supports the complete Avro specification including complex types, logical types, and schema evolution.

/**
 * Parse a schema and return the corresponding type or protocol
 * @param {Object|String} schema - Schema definition, JSON string, or file path
 * @param {Object} opts - Optional parsing options
 * @returns {Type|Protocol} Type instance or Protocol instance
 */
function parse(schema, opts);

/**
 * Abstract base class for all Avro types
 */
class Type {
 /**
  * Serialize value to buffer
  * @param {*} val - Value to serialize
  * @returns {Buffer} Serialized data
  */
 toBuffer(val);

 /**
  * Deserialize from buffer
  * @param {Buffer} buf - Buffer to deserialize
  * @param {Object} resolver - Optional resolver for schema evolution
  * @param {Boolean} noCheck - Skip validation
  * @returns {*} Deserialized value
  */
 fromBuffer(buf, resolver, noCheck);

 /**
  * Validate value against schema
  * @param {*} val - Value to validate
  * @param {Object} opts - Validation options
  * @returns {Boolean} True if valid
  */
 isValid(val, opts);
}

Schema Parsing and Types

File Operations

Avro container file processing with support for compression codecs, metadata extraction, and streaming operations. Enables reading and writing standard Avro data files with full header and block processing.

/**
 * Create readable stream for decoding Avro container files
 * @param {String} path - Path to Avro file
 * @param {Object} opts - Options object
 * @returns {ReadableStream} Stream of decoded records
 */
function createFileDecoder(path, opts);

/**
 * Create writable stream for encoding Avro container files
 * @param {String} path - Output file path
 * @param {Object|Type} schema - Schema or Type instance
 * @param {Object} opts - Options object
 * @returns {WritableStream} Stream for writing records
 */
function createFileEncoder(path, schema, opts);

/**
 * Extract header metadata from Avro container file
 * @param {String} path - Path to Avro file
 * @param {Object} opts - Options object
 * @returns {Object} Header object with metadata
 */
function extractFileHeader(path, opts);

File Operations

Protocol Communication

Avro protocol implementation for RPC/IPC communication with support for stateful and stateless emitters/listeners. Provides handshake negotiation, message routing, and bidirectional communication.

// Parse function can return Protocol instances
const protocolOrType = parse(schemaOrProtocolDef, opts);

/**
 * Represents an Avro protocol (returned by parse() for protocol definitions)
 */
class Protocol {
 /**
  * Emit message via emitter
  * @param {String} name - Message name
  * @param {Object} req - Request data
  * @param {MessageEmitter} emitter - Message emitter
  * @param {Function} cb - Callback function
  */
 emit(name, req, emitter, cb);

 /**
  * Create message emitter
  * @param {Object} transport - Transport object
  * @param {Object} opts - Options object
  * @param {Function} cb - Callback function
  * @returns {MessageEmitter} Message emitter instance
  */
 createEmitter(transport, opts, cb);
}

Protocol Communication

Streaming

Node.js streaming interfaces for processing Avro data with support for raw value encoding/decoding and container file block processing. Enables efficient handling of large datasets.

/**
 * Duplex stream for decoding raw Avro data
 */
class RawDecoder extends stream.Duplex {
 constructor(schema, opts);
}

/**
 * Transform stream for encoding raw Avro data
 */
class RawEncoder extends stream.Transform {
 constructor(schema, opts);
}

/**
 * Duplex stream for decoding Avro container files
 */
class BlockDecoder extends stream.Duplex {
 constructor(opts);
 static getDefaultCodecs();
}

Streaming

Deprecated Validation

Legacy validation APIs for backward compatibility with older versions. These APIs are deprecated in favor of the modern Type-based validation system.

/**
 * @deprecated Use Type.isValid() instead
 * Schema validator for validating JavaScript objects against Avro schemas
 */
class Validator {
  constructor(schema, namespace, namedTypes);
  
  /**
   * Validate object against schema
   * @param {*} obj - Object to validate
   * @returns {boolean} True if valid
   */
  validate(obj);
  
  /**
   * Static validation method
   * @param {Object} schema - Avro schema
   * @param {*} obj - Object to validate
   * @returns {boolean} True if valid
   */
  static validate(schema, obj);
}

/**
 * @deprecated Use Protocol-based validation instead
 * Protocol validator for validating objects against protocol-defined types
 */
class ProtocolValidator {
  constructor(protocol);
  
  /**
   * Validate object against protocol type
   * @param {string} typeName - Name of type in protocol
   * @param {*} obj - Object to validate
   * @returns {boolean} True if valid
   */
  validate(typeName, obj);
  
  /**
   * Static validation method
   * @param {Object} protocol - Protocol definition
   * @param {string} typeName - Name of type in protocol
   * @param {*} obj - Object to validate
   * @returns {boolean} True if valid
   */
  static validate(protocol, typeName, obj);
}

Utility Functions

Low-level utility functions for data manipulation, random generation, and buffer operations. Most users will not need these directly.

/**
 * Linear congruential generator for deterministic random data
 */
class Lcg {
  constructor(seed);
  nextBoolean();
  nextInt(start, end);
  nextFloat(start, end);
  nextString(len, flags);
  nextBuffer(len);
  choice(arr);
}

/**
 * Ordered queue for managing indexed items
 */
class OrderedQueue {
  constructor();
  push(item);
  pop();
}

/**
 * Buffer reading/writing abstraction with position tracking
 */
class Tap {
  constructor(buf, pos);
  isValid();
  getValue();
  // Read/write methods for Avro types
  readBoolean();
  writeBoolean(b);
  readInt();
  writeInt(n);
  readFloat();
  writeFloat(f);
  readDouble();
  writeDouble(d);
  readString();
  writeString(s);
  readBytes();
  writeBytes(buf);
  // Additional utility methods...
}

Utility Functions

Common Types

interface ParseOptions {
  /** Parent namespace for type resolution */
  namespace?: string;
  /** Registry of predefined types */
  registry?: Object;
  /** Dictionary of LogicalType classes */
  logicalTypes?: Object;
  /** Throw error on invalid logical types */
  assertLogicalTypes?: boolean;
  /** Custom type creation hook */
  typeHook?: (attrs: Object, opts: ParseOptions) => Type;
}

interface FileOptions {
  /** Whether to decode values (default: true) */
  decode?: boolean;
  /** Compression codecs */
  codecs?: Object;
  /** Schema parsing options */
  parseOpts?: ParseOptions;
  /** Block size for encoding */
  blockSize?: number;
  /** Compression codec name */
  codec?: string;
  /** Omit file header */
  omitHeader?: boolean;
  /** Custom sync marker */
  syncMarker?: Buffer;
}