Avro container file processing with support for compression codecs, metadata extraction, and streaming operations. Enables reading and writing standard Avro data files with full header and block processing.
Create readable streams for decoding Avro container files.
/**
* Create readable stream for decoding Avro container files
* @param {String} path - Path to Avro file
* @param {Object} opts - Options object
* @returns {ReadableStream} Stream of decoded records
*/
function createFileDecoder(path, opts);Usage Examples:
const avro = require('avro-js');
// Basic file decoding
const decoder = avro.createFileDecoder('./data.avro');
decoder.on('data', (record) => {
console.log(record);
});
// Decoding with options
const decoderWithOpts = avro.createFileDecoder('./data.avro', {
decode: true, // Decode to JavaScript objects (default: true)
codecs: { // Custom decompression codecs
'snappy': require('snappy').uncompress
}
});
// Get raw buffers instead of decoded objects
const rawDecoder = avro.createFileDecoder('./data.avro', {
decode: false
});Create writable streams for encoding Avro container files.
/**
* Create writable stream for encoding Avro container files
* @param {String} path - Output file path
* @param {Object|Type} schema - Schema or Type instance
* @param {Object} opts - Options object
* @returns {WritableStream} Stream for writing records
*/
function createFileEncoder(path, schema, opts);Usage Examples:
const avro = require('avro-js');
const schema = {
type: 'record',
name: 'User',
fields: [
{ name: 'id', type: 'int' },
{ name: 'name', type: 'string' }
]
};
// Basic file encoding
const encoder = avro.createFileEncoder('./output.avro', schema);
encoder.write({ id: 1, name: 'Alice' });
encoder.write({ id: 2, name: 'Bob' });
encoder.end();
// Encoding with compression
const compressedEncoder = avro.createFileEncoder('./output.avro', schema, {
codec: 'deflate',
blockSize: 1024 * 64 // 64KB blocks
});
// Custom sync marker
const customEncoder = avro.createFileEncoder('./output.avro', schema, {
syncMarker: Buffer.from('CustomSync123456')
});Extract metadata from Avro container file headers.
/**
* Extract header metadata from Avro container file
* @param {String} path - Path to Avro file
* @param {Object} opts - Options object
* @returns {Object} Header object with metadata
*/
function extractFileHeader(path, opts);Usage Examples:
const avro = require('avro-js');
// Extract file header
const header = avro.extractFileHeader('./data.avro');
console.log(header.meta); // File metadata
console.log(header.schema); // Schema used in file
console.log(header.codec); // Compression codec
// Extract with size limit
const limitedHeader = avro.extractFileHeader('./data.avro', {
size: 1024 // Only read first 1KB
});Process raw Avro data without container file format.
/**
* Duplex stream for decoding raw Avro data
*/
class RawDecoder extends stream.Duplex {
/**
* Create raw decoder for concatenated Avro values
* @param {Object|Type} schema - Schema or Type instance
* @param {Object} opts - Options object
*/
constructor(schema, opts);
}
/**
* Transform stream for encoding raw Avro data
*/
class RawEncoder extends stream.Transform {
/**
* Create raw encoder for concatenated Avro values
* @param {Object|Type} schema - Schema or Type instance
* @param {Object} opts - Options object
*/
constructor(schema, opts);
}Usage Examples:
const avro = require('avro-js');
const { RawDecoder, RawEncoder } = avro.streams;
const schema = { type: 'string' };
// Encode raw values
const encoder = new RawEncoder(schema);
encoder.write('Hello');
encoder.write('World');
encoder.end();
// Decode raw values
const decoder = new RawDecoder(schema);
decoder.on('data', (value) => {
console.log(value); // 'Hello', 'World'
});
// Pipe encoded data to decoder
encoder.pipe(decoder);Process Avro container files with block-level control.
/**
* Duplex stream for decoding Avro container files
*/
class BlockDecoder extends stream.Duplex {
/**
* Create block decoder for container files
* @param {Object} opts - Options object
*/
constructor(opts);
/**
* Get default compression codecs
* @returns {Object} Default codecs (null, deflate)
*/
static getDefaultCodecs();
}
/**
* Duplex stream for encoding Avro container files
*/
class BlockEncoder extends stream.Duplex {
/**
* Create block encoder for container files
* @param {Object|Type} schema - Schema or Type instance
* @param {Object} opts - Options object
*/
constructor(schema, opts);
/**
* Get default compression codecs
* @returns {Object} Default codecs (null, deflate)
*/
static getDefaultCodecs();
/**
* Get downstream writable stream
* @returns {WritableStream} Downstream stream
*/
getDownstream();
}Usage Examples:
const avro = require('avro-js');
const { BlockDecoder, BlockEncoder } = avro.streams;
const fs = require('fs');
// Block-level decoding
const blockDecoder = new BlockDecoder({
codecs: {
'null': (buf, cb) => cb(null, buf),
'deflate': require('zlib').inflateRaw,
'snappy': require('snappy').uncompress
}
});
blockDecoder.on('metadata', (type, codec, header) => {
console.log('File metadata:', header.meta);
console.log('Compression:', codec);
});
fs.createReadStream('./data.avro').pipe(blockDecoder);
// Block-level encoding
const schema = { type: 'int' };
const blockEncoder = new BlockEncoder(schema, {
codec: 'deflate',
blockSize: 1000,
codecs: BlockEncoder.getDefaultCodecs()
});
blockEncoder.pipe(fs.createWriteStream('./output.avro'));
blockEncoder.write(42);
blockEncoder.write(100);
blockEncoder.end();/** Avro file magic bytes */
const MAGIC_BYTES = Buffer.from('Obj\x01');
/** Avro file header type definition */
const HEADER_TYPE = {
type: 'record',
name: 'org.apache.avro.file.Header',
fields: [
{ name: 'magic', type: { type: 'fixed', name: 'Magic', size: 4 } },
{ name: 'meta', type: { type: 'map', values: 'bytes' } },
{ name: 'sync', type: { type: 'fixed', name: 'Sync', size: 16 } }
]
};
/** Avro file block type definition */
const BLOCK_TYPE = {
type: 'record',
name: 'org.apache.avro.file.Block',
fields: [
{ name: 'count', type: 'long' },
{ name: 'data', type: 'bytes' },
{ name: 'sync', type: { type: 'fixed', name: 'Sync', size: 16 } }
]
};interface FileDecoderOptions {
/** Whether to decode values to JavaScript objects (default: true) */
decode?: boolean;
/** Custom compression codecs */
codecs?: {
[codecName: string]: (buf: Buffer, cb: (err: Error, result: Buffer) => void) => void;
};
/** Schema parsing options */
parseOpts?: ParseOptions;
}
interface FileEncoderOptions {
/** Block size in bytes (default: 65536) */
blockSize?: number;
/** Compression codec name (default: 'null') */
codec?: string;
/** Custom compression codecs */
codecs?: {
[codecName: string]: (buf: Buffer, cb: (err: Error, result: Buffer) => void) => void;
};
/** Omit file header for raw block output */
omitHeader?: boolean;
/** Custom 16-byte sync marker */
syncMarker?: Buffer;
/** Metadata to include in file header */
metadata?: { [key: string]: Buffer };
}
interface HeaderExtractionOptions {
/** Whether to decode metadata values */
decode?: boolean;
/** Maximum bytes to read for header */
size?: number;
}
interface RawStreamOptions {
/** Whether to decode values to JavaScript objects (default: true) */
decode?: boolean;
/** Whether to batch writes */
batchWrites?: boolean;
}Constants used for Avro container file format identification and processing.
/**
* Magic bytes that identify Avro container files ('Obj\x01')
* Used to validate file headers and detect Avro format
*/
const MAGIC_BYTES: Buffer;
/**
* Internal header type schema used for container file headers
* Defines the structure of Avro file metadata
*/
const HEADER_TYPE: Type;These constants are primarily used internally but are exposed for testing and advanced use cases where direct file format manipulation is needed.