CtrlK
BlogDocsLog inGet started
Tessl Logo

tessl/npm-mux-js

Lightweight utilities for inspecting and manipulating video container formats

—

Pending
Quality

Pending

Does it follow best practices?

Impact

Pending

No eval scenarios have been run

SecuritybySnyk

Pending

The risk profile of this skill

Overview
Eval results
Files

captions.mddocs/

Caption Processing

Comprehensive caption support for CEA-608, CEA-708, and WebVTT formats. The caption processing capabilities extract and parse caption data from video streams for accessibility compliance and subtitle display.

Capabilities

Caption Parser (CEA-608)

Parser for extracting CEA-608 closed caption data from MP4 streams. CEA-608 is the standard for analog television closed captions, commonly used in digital streams for compatibility.

/**
 * Parser for CEA-608 captions embedded in MP4 streams
 * Extracts caption data from video samples and converts to text cues
 */
class CaptionParser {
  constructor();
  
  /** Parse caption data from MP4 video samples */
  parse(data: Uint8Array): Caption[];
  
  /** Clear all parsed captions from memory */
  clearParsedCaptions(): void;
  
  /** Get all currently parsed captions */
  getParsedCaptions(): Caption[];
}

interface Caption {
  startTime: number;
  endTime: number;
  text: string;
  line: number;
  position: number;
}

Usage Examples:

const muxjs = require("mux.js");

// Create caption parser
const captionParser = new muxjs.mp4.CaptionParser();

// Parse captions from MP4 video track
const mp4VideoData = new Uint8Array([/* MP4 video track data */]);
const captions = captionParser.parse(mp4VideoData);

// Process parsed captions
captions.forEach(caption => {
  console.log(`Caption: "${caption.text}"`);
  console.log(`Time: ${caption.startTime}s - ${caption.endTime}s`);
  console.log(`Position: line ${caption.line}, position ${caption.position}`);
  
  // Create VTT cue for display
  const cue = new VTTCue(caption.startTime, caption.endTime, caption.text);
  cue.line = caption.line;
  cue.position = caption.position;
  
  // Add to text track
  textTrack.addCue(cue);
});

// Clear parser state for next segment
captionParser.clearParsedCaptions();

WebVTT Parser

Parser for extracting WebVTT caption data from MP4 streams. WebVTT is the modern web standard for captions and subtitles.

/**
 * Parser for WebVTT captions in MP4 streams
 * Handles WebVTT cue parsing and timing information
 */
class WebVttParser {
  constructor();
  
  /** Parse WebVTT data from MP4 stream */
  parse(data: Uint8Array): WebVttCue[];
}

interface WebVttCue {
  startTime: number;
  endTime: number;
  text: string;
  id?: string;
  settings?: string;
}

Usage Examples:

const muxjs = require("mux.js");

// Create WebVTT parser
const webvttParser = new muxjs.mp4.WebVttParser();

// Parse WebVTT from MP4 subtitle track
const mp4SubtitleData = new Uint8Array([/* MP4 subtitle track data */]);
const webvttCues = webvttParser.parse(mp4SubtitleData);

// Process WebVTT cues
webvttCues.forEach(cue => {
  console.log(`WebVTT Cue: "${cue.text}"`);
  console.log(`Time: ${cue.startTime}s - ${cue.endTime}s`);
  if (cue.id) console.log(`ID: ${cue.id}`);
  if (cue.settings) console.log(`Settings: ${cue.settings}`);
  
  // Create VTT cue for display
  const vttCue = new VTTCue(cue.startTime, cue.endTime, cue.text);
  if (cue.id) vttCue.id = cue.id;
  
  // Apply settings if present
  if (cue.settings) {
    // Parse settings string (e.g., "line:90% position:50%")
    parseVttSettings(cue.settings, vttCue);
  }
  
  textTrack.addCue(vttCue);
});

Transport Stream Caption Processing

Caption processing directly from MPEG-2 transport streams, handling embedded caption data in video elementary streams.

/**
 * Main caption stream processor for transport streams
 * Handles CEA-608 and CEA-708 caption data embedded in video streams
 */
class CaptionStream {
  constructor();
  
  /** Process caption data from video elementary stream */
  push(data: Uint8Array): void;
  
  /** Flush any buffered caption data */
  flush(): void;
  
  /** Reset caption parser state */
  reset(): void;
  
  /** Register event listeners */
  on(event: 'data', callback: (captionSet: CaptionSet) => void): void;
  on(event: 'done', callback: () => void): void;
}

interface CaptionSet {
  startTime: number;
  endTime: number;
  content: Caption[];
  ccData: Uint8Array;
}

CEA-608 Stream Processing

Specialized processor for CEA-608 closed captions from transport streams.

/**
 * CEA-608 closed caption processor
 * Handles line 21 caption data and character decoding
 */
class Cea608Stream {
  constructor();
  
  /** Process CEA-608 caption data */
  push(data: CaptionData): void;
  
  /** Flush buffered caption text */
  flush(): void;
  
  /** Reset parser state */
  reset(): void;
  
  /** Register event listeners */
  on(event: 'data', callback: (caption: Cea608Caption) => void): void;
}

interface CaptionData {
  pts: number;
  dts: number;
  data: Uint8Array;
  stream: string;
}

interface Cea608Caption {
  startTime: number;
  endTime: number;
  text: string;
  line: number;
  position: number;
  channel: number;
  field: number;
}

CEA-708 Stream Processing

Specialized processor for CEA-708 closed captions from transport streams.

/**
 * CEA-708 closed caption processor
 * Handles digital television caption services and windowing
 */
class Cea708Stream {
  constructor();
  
  /** Process CEA-708 caption data */
  push(data: CaptionData): void;
  
  /** Flush buffered caption data */
  flush(): void;
  
  /** Register event listeners */
  on(event: 'data', callback: (caption: Cea708Caption) => void): void;
}

interface Cea708Caption {
  startTime: number;
  endTime: number;
  text: string;
  windowId: number;
  serviceNumber: number;
  priority: number;
  anchorId?: number;
  relativePositioning?: boolean;
  anchorVertical?: number;
  anchorHorizontal?: number;
}

Caption Pipeline Integration

Integration with mux.js transmuxing pipeline for automatic caption extraction.

/**
 * Caption processing is integrated into the main transmuxer
 * Captions are automatically extracted during transmuxing
 */
interface TransmuxedSegment {
  initSegment: Uint8Array;
  data: Uint8Array;
  metadata: {
    frames: ID3Frame[];
  };
  /** Parsed caption data from the segment */
  captions: CaptionSet[];
}

Usage Examples:

const muxjs = require("mux.js");

// Captions are automatically processed during transmuxing
const transmuxer = new muxjs.mp4.Transmuxer();

transmuxer.on('data', (segment) => {
  console.log(`Found ${segment.captions.length} caption sets`);
  
  // Process captions
  segment.captions.forEach(captionSet => {
    console.log(`Caption set: ${captionSet.startTime}s - ${captionSet.endTime}s`);
    
    captionSet.content.forEach(caption => {
      console.log(`  "${caption.text}"`);
      
      // Create VTT cue
      const cue = new VTTCue(caption.startTime, caption.endTime, caption.text);
      cue.line = caption.line;
      cue.position = caption.position;
      
      // Add to appropriate text track
      textTrack.addCue(cue);
    });
  });
  
  // Process video/audio data
  sourceBuffer.appendBuffer(segment.data);
});

Manual Caption Processing Pipeline

For more control over caption processing, you can create a manual pipeline.

const muxjs = require("mux.js");

// Create caption processing pipeline
const packetStream = new muxjs.mp2t.TransportPacketStream();
const parseStream = new muxjs.mp2t.TransportParseStream();
const elementaryStream = new muxjs.mp2t.ElementaryStream();
const captionStream = new muxjs.mp2t.CaptionStream();
const cea608Stream = new muxjs.mp2t.Cea608Stream();
const cea708Stream = new muxjs.mp2t.Cea708Stream();

// Connect pipeline
packetStream
  .pipe(parseStream)
  .pipe(elementaryStream);

// Handle video data with captions
elementaryStream.on('data', (pesPacket) => {
  if (pesPacket.type === 'video') {
    captionStream.push(pesPacket.data);
  }
});

// Handle caption data
captionStream.on('data', (captionSet) => {
  // Route to appropriate caption processor
  captionSet.content.forEach(caption => {
    if (caption.field === 1) {
      // CEA-608 Field 1
      cea608Stream.push({
        pts: captionSet.startTime,
        dts: captionSet.startTime,
        data: captionSet.ccData,
        stream: 'cc1'
      });
    }
  });
});

// Handle processed captions
cea608Stream.on('data', (caption) => {
  console.log(`CEA-608 Caption: "${caption.text}"`);
  console.log(`Channel: ${caption.channel}, Field: ${caption.field}`);
  
  // Create text track cue
  const cue = new VTTCue(caption.startTime, caption.endTime, caption.text);
  textTrack.addCue(cue);
});

cea708Stream.on('data', (caption) => {
  console.log(`CEA-708 Caption: "${caption.text}"`);
  console.log(`Service: ${caption.serviceNumber}, Window: ${caption.windowId}`);
});

Caption Utilities

Utility functions for working with caption data.

// Example utility functions (these would be implemented in your application)

/**
 * Parse WebVTT settings string and apply to VTTCue
 */
function parseVttSettings(settings, cue) {
  const pairs = settings.split(' ');
  pairs.forEach(pair => {
    const [key, value] = pair.split(':');
    switch (key) {
      case 'line':
        cue.line = parseFloat(value);
        break;
      case 'position':
        cue.position = parseFloat(value);
        break;
      case 'size':
        cue.size = parseFloat(value);
        break;
      case 'align':
        cue.align = value;
        break;
    }
  });
}

/**
 * Convert timestamp from 90kHz to seconds
 */
function timestampToSeconds(timestamp) {
  return timestamp / 90000;
}

/**
 * Format caption text for display
 */
function formatCaptionText(text) {
  // Handle special characters and formatting
  return text
    .replace(/\n/g, ' ')
    .replace(/\s+/g, ' ')
    .trim();
}

Types

interface Caption {
  startTime: number;
  endTime: number;
  text: string;
  line: number;
  position: number;
}

interface CaptionSet {
  startTime: number;
  endTime: number;
  content: Caption[];
  ccData: Uint8Array;
}

interface WebVttCue {
  startTime: number;
  endTime: number;
  text: string;
  id?: string;
  settings?: string;
}

interface Cea608Caption {
  startTime: number;
  endTime: number;
  text: string;
  line: number;
  position: number;
  channel: number;
  field: number;
}

interface Cea708Caption {
  startTime: number;
  endTime: number;
  text: string;
  windowId: number;
  serviceNumber: number;
  priority: number;
  anchorId?: number;
  relativePositioning?: boolean;
  anchorVertical?: number;
  anchorHorizontal?: number;
}

interface CaptionData {
  pts: number;
  dts: number;
  data: Uint8Array;
  stream: string;
}

docs

captions.md

codecs.md

flv.md

index.md

mp2t.md

mp4.md

tools.md

tile.json