or run

npx @tessl/cli init
Log in

Version

Tile

Overview

Evals

Files

docs

captions.mdcodecs.mdflv.mdindex.mdmp2t.mdmp4.mdtools.md
tile.json

codecs.mddocs/

Codec Processing

Specialized codec parsers for H.264 video and ADTS/AAC audio streams. The codecs module handles bitstream parsing, NAL unit extraction, and audio frame processing within the mux.js streaming pipeline.

Capabilities

ADTS Audio Processing

Parser for ADTS (Audio Data Transport Stream) format that extracts AAC audio frames from transport streams.

/**
 * ADTS stream parser for AAC audio processing
 * Accepts ElementaryStream input and emits parsed AAC audio frames
 */
class Adts {
  constructor();
  
  /** Process ADTS data and emit AAC frames */
  push(data: Uint8Array): void;
  
  /** Flush any remaining data */
  flush(): void;
  
  /** Reset parser state */
  reset(): void;
  
  /** Register event listeners */
  on(event: 'data', callback: (frame: AACFrame) => void): void;
  on(event: 'done', callback: () => void): void;
}

interface AACFrame {
  data: Uint8Array;
  pts: number;
  dts: number;
  sampleRate: number;
  channelCount: number;
  sampleCount: number;
}

Usage Examples:

const muxjs = require("mux.js");

// Create ADTS parser
const adtsParser = new muxjs.codecs.Adts();

// Handle parsed AAC frames
adtsParser.on('data', (frame) => {
  console.log(`AAC frame: ${frame.data.length} bytes`);
  console.log(`Sample rate: ${frame.sampleRate}Hz`);
  console.log(`Channels: ${frame.channelCount}`);
  console.log(`PTS: ${frame.pts}`);
  
  // Process AAC frame data
  processAACFrame(frame);
});

// Process ADTS data from elementary stream
adtsParser.push(adtsData);
adtsParser.flush();

H.264 Video Processing

Comprehensive H.264 bitstream processing including NAL unit parsing and video frame extraction.

const h264: {
  /** H.264 stream parser for video processing */
  H264Stream: class H264Stream {
    constructor();
    
    /** Process H.264 data and emit video frames */
    push(data: Uint8Array): void;
    
    /** Flush any remaining data */
    flush(): void;
    
    /** Reset parser state */
    reset(): void;
    
    /** Register event listeners */
    on(event: 'data', callback: (frame: H264Frame) => void): void;
  };
  
  /** NAL unit byte stream parser */
  NalByteStream: class NalByteStream {
    constructor();
    
    /** Process NAL byte stream and emit NAL units */
    push(data: Uint8Array): void;
    
    /** Flush any remaining data */
    flush(): void;
    
    /** Register event listeners */
    on(event: 'data', callback: (nalUnit: NALUnit) => void): void;
  };
};

interface H264Frame {
  data: Uint8Array;
  pts: number;
  dts: number;
  keyframe: boolean;
  nalUnits: NALUnit[];
}

interface NALUnit {
  data: Uint8Array;
  type: number;
  nalUnitType: string;
}

Usage Examples:

const muxjs = require("mux.js");

// Create H.264 stream parser
const h264Stream = new muxjs.codecs.h264.H264Stream();

// Handle parsed video frames
h264Stream.on('data', (frame) => {
  console.log(`H.264 frame: ${frame.data.length} bytes`);
  console.log(`Keyframe: ${frame.keyframe}`);
  console.log(`NAL units: ${frame.nalUnits.length}`);
  console.log(`PTS: ${frame.pts}, DTS: ${frame.dts}`);
  
  // Process video frame
  processVideoFrame(frame);
});

// Create NAL byte stream parser
const nalStream = new muxjs.codecs.h264.NalByteStream();

// Handle parsed NAL units
nalStream.on('data', (nalUnit) => {
  console.log(`NAL unit type: ${nalUnit.nalUnitType} (${nalUnit.type})`);
  console.log(`Data length: ${nalUnit.data.length} bytes`);
  
  // Process NAL unit
  processNALUnit(nalUnit);
});

// Chain parsers together
nalStream.pipe(h264Stream);

// Process H.264 bitstream data
nalStream.push(h264BitstreamData);
nalStream.flush();

Codec Pipeline Integration

The codec parsers integrate seamlessly with the mux.js streaming pipeline architecture.

/**
 * Base Stream class that all codec parsers extend
 */
class Stream {
  /** Connect this stream to another stream */
  pipe(destination: Stream): Stream;
  
  /** Register event listener */
  on(event: string, callback: Function): void;
  
  /** Remove event listener */
  off(event?: string): void;
  
  /** Trigger event */
  trigger(event: string, ...args: any[]): void;
}

Usage Examples:

const muxjs = require("mux.js");

// Create processing pipeline
const transportPacketStream = new muxjs.mp2t.TransportPacketStream();
const transportParseStream = new muxjs.mp2t.TransportParseStream();
const elementaryStream = new muxjs.mp2t.ElementaryStream();
const adtsParser = new muxjs.codecs.Adts();
const h264Parser = new muxjs.codecs.h264.H264Stream();

// Connect pipeline for audio processing
transportPacketStream
  .pipe(transportParseStream)
  .pipe(elementaryStream);

// Handle elementary stream data
elementaryStream.on('data', (data) => {
  if (data.streamType === muxjs.mp2t.ADTS_STREAM_TYPE) {
    adtsParser.push(data.data);
  } else if (data.streamType === muxjs.mp2t.H264_STREAM_TYPE) {
    h264Parser.push(data.data);
  }
});

// Process transport stream
transportPacketStream.push(transportStreamData);
transportPacketStream.flush();

Advanced Codec Features

AAC Configuration Detection

The ADTS parser automatically detects AAC configuration from the bitstream.

// ADTS parser automatically extracts audio configuration
adtsParser.on('data', (frame) => {
  // Audio configuration is available in frame properties
  const config = {
    sampleRate: frame.sampleRate,    // e.g., 44100, 48000
    channelCount: frame.channelCount, // e.g., 1, 2, 6
    profile: frame.profile,          // AAC profile
    sampleCount: frame.sampleCount   // Samples per frame (typically 1024)
  };
  
  console.log('AAC Configuration:', config);
});

H.264 NAL Unit Types

The H.264 parser identifies different NAL unit types for proper processing.

// NAL unit types commonly encountered
const NAL_UNIT_TYPES = {
  1: 'NON_IDR_PICTURE',
  5: 'IDR_PICTURE', 
  6: 'SEI',
  7: 'SPS',  // Sequence Parameter Set
  8: 'PPS',  // Picture Parameter Set
  9: 'ACCESS_UNIT_DELIMITER'
};

nalStream.on('data', (nalUnit) => {
  const typeName = NAL_UNIT_TYPES[nalUnit.type] || 'UNKNOWN';
  console.log(`NAL Unit: ${typeName} (${nalUnit.type})`);
  
  // Handle specific NAL unit types
  switch (nalUnit.type) {
    case 7: // SPS
      console.log('Found Sequence Parameter Set');
      break;
    case 8: // PPS  
      console.log('Found Picture Parameter Set');
      break;
    case 5: // IDR
      console.log('Found keyframe (IDR)');
      break;
  }
});

Types

interface CodecFrame {
  data: Uint8Array;
  pts: number;
  dts: number;
}

interface AACFrame extends CodecFrame {
  sampleRate: number;
  channelCount: number;
  sampleCount: number;
  profile?: number;
}

interface H264Frame extends CodecFrame {
  keyframe: boolean;
  nalUnits: NALUnit[];
}

interface NALUnit {
  data: Uint8Array;
  type: number;
  nalUnitType: string;
}

interface StreamEvent {
  type: 'video' | 'audio';
  data: Uint8Array;
  pts?: number;
  dts?: number;
  streamType: number;
}