0
# Caption Processing
1

2
Comprehensive caption support for CEA-608, CEA-708, and WebVTT formats. The caption processing capabilities extract and parse caption data from video streams for accessibility compliance and subtitle display.
3

4
## Capabilities
5

6
### Caption Parser (CEA-608)
7

8
Parser for extracting CEA-608 closed caption data from MP4 streams. CEA-608 is the standard for analog television closed captions, commonly used in digital streams for compatibility.
9

10
```javascript { .api }
11
/**
12
 * Parser for CEA-608 captions embedded in MP4 streams
13
 * Extracts caption data from video samples and converts to text cues
14
 */
15
class CaptionParser {
16
  constructor();
17
  
18
  /** Parse caption data from MP4 video samples */
19
  parse(data: Uint8Array): Caption[];
20
  
21
  /** Clear all parsed captions from memory */
22
  clearParsedCaptions(): void;
23
  
24
  /** Get all currently parsed captions */
25
  getParsedCaptions(): Caption[];
26
}
27

28
interface Caption {
29
  startTime: number;
30
  endTime: number;
31
  text: string;
32
  line: number;
33
  position: number;
34
}
35
```
36

37
**Usage Examples:**
38

39
```javascript
40
const muxjs = require("mux.js");
41

42
// Create caption parser
43
const captionParser = new muxjs.mp4.CaptionParser();
44

45
// Parse captions from MP4 video track
46
const mp4VideoData = new Uint8Array([/* MP4 video track data */]);
47
const captions = captionParser.parse(mp4VideoData);
48

49
// Process parsed captions
50
captions.forEach(caption => {
51
  console.log(`Caption: "${caption.text}"`);
52
  console.log(`Time: ${caption.startTime}s - ${caption.endTime}s`);
53
  console.log(`Position: line ${caption.line}, position ${caption.position}`);
54
  
55
  // Create VTT cue for display
56
  const cue = new VTTCue(caption.startTime, caption.endTime, caption.text);
57
  cue.line = caption.line;
58
  cue.position = caption.position;
59
  
60
  // Add to text track
61
  textTrack.addCue(cue);
62
});
63

64
// Clear parser state for next segment
65
captionParser.clearParsedCaptions();
66
```
67

68
### WebVTT Parser
69

70
Parser for extracting WebVTT caption data from MP4 streams. WebVTT is the modern web standard for captions and subtitles.
71

72
```javascript { .api }
73
/**
74
 * Parser for WebVTT captions in MP4 streams
75
 * Handles WebVTT cue parsing and timing information
76
 */
77
class WebVttParser {
78
  constructor();
79
  
80
  /** Parse WebVTT data from MP4 stream */
81
  parse(data: Uint8Array): WebVttCue[];
82
}
83

84
interface WebVttCue {
85
  startTime: number;
86
  endTime: number;
87
  text: string;
88
  id?: string;
89
  settings?: string;
90
}
91
```
92

93
**Usage Examples:**
94

95
```javascript
96
const muxjs = require("mux.js");
97

98
// Create WebVTT parser
99
const webvttParser = new muxjs.mp4.WebVttParser();
100

101
// Parse WebVTT from MP4 subtitle track
102
const mp4SubtitleData = new Uint8Array([/* MP4 subtitle track data */]);
103
const webvttCues = webvttParser.parse(mp4SubtitleData);
104

105
// Process WebVTT cues
106
webvttCues.forEach(cue => {
107
  console.log(`WebVTT Cue: "${cue.text}"`);
108
  console.log(`Time: ${cue.startTime}s - ${cue.endTime}s`);
109
  if (cue.id) console.log(`ID: ${cue.id}`);
110
  if (cue.settings) console.log(`Settings: ${cue.settings}`);
111
  
112
  // Create VTT cue for display
113
  const vttCue = new VTTCue(cue.startTime, cue.endTime, cue.text);
114
  if (cue.id) vttCue.id = cue.id;
115
  
116
  // Apply settings if present
117
  if (cue.settings) {
118
    // Parse settings string (e.g., "line:90% position:50%")
119
    parseVttSettings(cue.settings, vttCue);
120
  }
121
  
122
  textTrack.addCue(vttCue);
123
});
124
```
125

126
### Transport Stream Caption Processing
127

128
Caption processing directly from MPEG-2 transport streams, handling embedded caption data in video elementary streams.
129

130
```javascript { .api }
131
/**
132
 * Main caption stream processor for transport streams
133
 * Handles CEA-608 and CEA-708 caption data embedded in video streams
134
 */
135
class CaptionStream {
136
  constructor();
137
  
138
  /** Process caption data from video elementary stream */
139
  push(data: Uint8Array): void;
140
  
141
  /** Flush any buffered caption data */
142
  flush(): void;
143
  
144
  /** Reset caption parser state */
145
  reset(): void;
146
  
147
  /** Register event listeners */
148
  on(event: 'data', callback: (captionSet: CaptionSet) => void): void;
149
  on(event: 'done', callback: () => void): void;
150
}
151

152
interface CaptionSet {
153
  startTime: number;
154
  endTime: number;
155
  content: Caption[];
156
  ccData: Uint8Array;
157
}
158
```
159

160
### CEA-608 Stream Processing
161

162
Specialized processor for CEA-608 closed captions from transport streams.
163

164
```javascript { .api }
165
/**
166
 * CEA-608 closed caption processor
167
 * Handles line 21 caption data and character decoding
168
 */
169
class Cea608Stream {
170
  constructor();
171
  
172
  /** Process CEA-608 caption data */
173
  push(data: CaptionData): void;
174
  
175
  /** Flush buffered caption text */
176
  flush(): void;
177
  
178
  /** Reset parser state */
179
  reset(): void;
180
  
181
  /** Register event listeners */
182
  on(event: 'data', callback: (caption: Cea608Caption) => void): void;
183
}
184

185
interface CaptionData {
186
  pts: number;
187
  dts: number;
188
  data: Uint8Array;
189
  stream: string;
190
}
191

192
interface Cea608Caption {
193
  startTime: number;
194
  endTime: number;
195
  text: string;
196
  line: number;
197
  position: number;
198
  channel: number;
199
  field: number;
200
}
201
```
202

203
### CEA-708 Stream Processing
204

205
Specialized processor for CEA-708 closed captions from transport streams.
206

207
```javascript { .api }
208
/**
209
 * CEA-708 closed caption processor
210
 * Handles digital television caption services and windowing
211
 */
212
class Cea708Stream {
213
  constructor();
214
  
215
  /** Process CEA-708 caption data */
216
  push(data: CaptionData): void;
217
  
218
  /** Flush buffered caption data */
219
  flush(): void;
220
  
221
  /** Register event listeners */
222
  on(event: 'data', callback: (caption: Cea708Caption) => void): void;
223
}
224

225
interface Cea708Caption {
226
  startTime: number;
227
  endTime: number;
228
  text: string;
229
  windowId: number;
230
  serviceNumber: number;
231
  priority: number;
232
  anchorId?: number;
233
  relativePositioning?: boolean;
234
  anchorVertical?: number;
235
  anchorHorizontal?: number;
236
}
237
```
238

239
## Caption Pipeline Integration
240

241
Integration with mux.js transmuxing pipeline for automatic caption extraction.
242

243
```javascript { .api }
244
/**
245
 * Caption processing is integrated into the main transmuxer
246
 * Captions are automatically extracted during transmuxing
247
 */
248
interface TransmuxedSegment {
249
  initSegment: Uint8Array;
250
  data: Uint8Array;
251
  metadata: {
252
    frames: ID3Frame[];
253
  };
254
  /** Parsed caption data from the segment */
255
  captions: CaptionSet[];
256
}
257
```
258

259
**Usage Examples:**
260

261
```javascript
262
const muxjs = require("mux.js");
263

264
// Captions are automatically processed during transmuxing
265
const transmuxer = new muxjs.mp4.Transmuxer();
266

267
transmuxer.on('data', (segment) => {
268
  console.log(`Found ${segment.captions.length} caption sets`);
269
  
270
  // Process captions
271
  segment.captions.forEach(captionSet => {
272
    console.log(`Caption set: ${captionSet.startTime}s - ${captionSet.endTime}s`);
273
    
274
    captionSet.content.forEach(caption => {
275
      console.log(`  "${caption.text}"`);
276
      
277
      // Create VTT cue
278
      const cue = new VTTCue(caption.startTime, caption.endTime, caption.text);
279
      cue.line = caption.line;
280
      cue.position = caption.position;
281
      
282
      // Add to appropriate text track
283
      textTrack.addCue(cue);
284
    });
285
  });
286
  
287
  // Process video/audio data
288
  sourceBuffer.appendBuffer(segment.data);
289
});
290
```
291

292
### Manual Caption Processing Pipeline
293

294
For more control over caption processing, you can create a manual pipeline.
295

296
```javascript
297
const muxjs = require("mux.js");
298

299
// Create caption processing pipeline
300
const packetStream = new muxjs.mp2t.TransportPacketStream();
301
const parseStream = new muxjs.mp2t.TransportParseStream();
302
const elementaryStream = new muxjs.mp2t.ElementaryStream();
303
const captionStream = new muxjs.mp2t.CaptionStream();
304
const cea608Stream = new muxjs.mp2t.Cea608Stream();
305
const cea708Stream = new muxjs.mp2t.Cea708Stream();
306

307
// Connect pipeline
308
packetStream
309
  .pipe(parseStream)
310
  .pipe(elementaryStream);
311

312
// Handle video data with captions
313
elementaryStream.on('data', (pesPacket) => {
314
  if (pesPacket.type === 'video') {
315
    captionStream.push(pesPacket.data);
316
  }
317
});
318

319
// Handle caption data
320
captionStream.on('data', (captionSet) => {
321
  // Route to appropriate caption processor
322
  captionSet.content.forEach(caption => {
323
    if (caption.field === 1) {
324
      // CEA-608 Field 1
325
      cea608Stream.push({
326
        pts: captionSet.startTime,
327
        dts: captionSet.startTime,
328
        data: captionSet.ccData,
329
        stream: 'cc1'
330
      });
331
    }
332
  });
333
});
334

335
// Handle processed captions
336
cea608Stream.on('data', (caption) => {
337
  console.log(`CEA-608 Caption: "${caption.text}"`);
338
  console.log(`Channel: ${caption.channel}, Field: ${caption.field}`);
339
  
340
  // Create text track cue
341
  const cue = new VTTCue(caption.startTime, caption.endTime, caption.text);
342
  textTrack.addCue(cue);
343
});
344

345
cea708Stream.on('data', (caption) => {
346
  console.log(`CEA-708 Caption: "${caption.text}"`);
347
  console.log(`Service: ${caption.serviceNumber}, Window: ${caption.windowId}`);
348
});
349
```
350

351
## Caption Utilities
352

353
Utility functions for working with caption data.
354

355
```javascript
356
// Example utility functions (these would be implemented in your application)
357

358
/**
359
 * Parse WebVTT settings string and apply to VTTCue
360
 */
361
function parseVttSettings(settings, cue) {
362
  const pairs = settings.split(' ');
363
  pairs.forEach(pair => {
364
    const [key, value] = pair.split(':');
365
    switch (key) {
366
      case 'line':
367
        cue.line = parseFloat(value);
368
        break;
369
      case 'position':
370
        cue.position = parseFloat(value);
371
        break;
372
      case 'size':
373
        cue.size = parseFloat(value);
374
        break;
375
      case 'align':
376
        cue.align = value;
377
        break;
378
    }
379
  });
380
}
381

382
/**
383
 * Convert timestamp from 90kHz to seconds
384
 */
385
function timestampToSeconds(timestamp) {
386
  return timestamp / 90000;
387
}
388

389
/**
390
 * Format caption text for display
391
 */
392
function formatCaptionText(text) {
393
  // Handle special characters and formatting
394
  return text
395
    .replace(/\n/g, ' ')
396
    .replace(/\s+/g, ' ')
397
    .trim();
398
}
399
```
400

401
## Types
402

403
```javascript { .api }
404
interface Caption {
405
  startTime: number;
406
  endTime: number;
407
  text: string;
408
  line: number;
409
  position: number;
410
}
411

412
interface CaptionSet {
413
  startTime: number;
414
  endTime: number;
415
  content: Caption[];
416
  ccData: Uint8Array;
417
}
418

419
interface WebVttCue {
420
  startTime: number;
421
  endTime: number;
422
  text: string;
423
  id?: string;
424
  settings?: string;
425
}
426

427
interface Cea608Caption {
428
  startTime: number;
429
  endTime: number;
430
  text: string;
431
  line: number;
432
  position: number;
433
  channel: number;
434
  field: number;
435
}
436

437
interface Cea708Caption {
438
  startTime: number;
439
  endTime: number;
440
  text: string;
441
  windowId: number;
442
  serviceNumber: number;
443
  priority: number;
444
  anchorId?: number;
445
  relativePositioning?: boolean;
446
  anchorVertical?: number;
447
  anchorHorizontal?: number;
448
}
449

450
interface CaptionData {
451
  pts: number;
452
  dts: number;
453
  data: Uint8Array;
454
  stream: string;
455
}
456
```