0
# MP4 Processing
1

2
Complete MP4 container support including box generation, transmuxing from transport streams, audio/video segment processing, and caption parsing for fragmented MP4 suitable for Media Source Extensions.
3

4
## Capabilities
5

6
### MP4 Transmuxer
7

8
Main transmuxer class for converting MPEG-2 Transport Streams to fragmented MP4 format compatible with MSE (Media Source Extensions).
9

10
```javascript { .api }
11
/**
12
 * MP4 transmuxer for converting TS to fragmented MP4
13
 */
14
class Transmuxer extends Stream {
15
  constructor(options?: TransmuxerOptions);
16
  
17
  /** Process input TS bytes */
18
  push(bytes: Uint8Array): void;
19
  
20
  /** Complete processing and output final segments */
21
  flush(): void;
22
  
23
  /** Set base media decode time for timing alignment */
24
  setBaseMediaDecodeTime(time: number): void;
25
  
26
  /** Align GOP timing with another stream */
27
  alignGopsWith(stream: Transmuxer): void;
28
}
29

30
interface TransmuxerOptions {
31
  /** Keep original timestamps instead of rebasing */
32
  keepOriginalTimestamps?: boolean;
33
  /** Remove CEA-608 captions */
34
  remux?: boolean;
35
}
36
```
37

38
**Usage Example:**
39

40
```javascript
41
const muxjs = require("mux.js");
42

43
const transmuxer = new muxjs.mp4.Transmuxer();
44

45
transmuxer.on('data', function(segment) {
46
  // segment.initSegment - MP4 initialization segment (when present)
47
  // segment.data - MP4 media segment data
48
  
49
  if (segment.initSegment) {
50
    // First segment includes initialization data
51
    console.log('Init segment size:', segment.initSegment.byteLength);
52
    // Append to MediaSource buffer
53
    sourceBuffer.appendBuffer(segment.initSegment);
54
  }
55
  
56
  console.log('Media segment size:', segment.data.byteLength);
57
  sourceBuffer.appendBuffer(segment.data);
58
});
59

60
// Process transport stream data
61
transmuxer.push(tsBytes);
62
transmuxer.flush();
63
```
64

65
### Audio and Video Segment Streams
66

67
Specialized streams for processing audio and video segments within the transmuxing pipeline.
68

69
```javascript { .api }
70
/**
71
 * Audio segment processor for MP4 output
72
 */
73
class AudioSegmentStream extends Stream {
74
  constructor(track: AudioTrack, options?: SegmentOptions);
75
  
76
  /** Process audio data */
77
  push(data: AudioData): void;
78
  
79
  /** Flush audio segment */
80
  flush(): void;
81
}
82

83
/**
84
 * Video segment processor for MP4 output  
85
 */
86
class VideoSegmentStream extends Stream {
87
  constructor(track: VideoTrack, options?: SegmentOptions);
88
  
89
  /** Process video data */
90
  push(data: VideoData): void;
91
  
92
  /** Flush video segment */
93
  flush(): void;
94
}
95

96
interface SegmentOptions {
97
  baseMediaDecodeTime?: number;
98
  alignGopsAtEnd?: boolean;
99
}
100
```
101

102
### MP4 Probing and Analysis
103

104
Utilities for analyzing MP4 structure, extracting metadata, and parsing timing information.
105

106
```javascript { .api }
107
/**
108
 * Find specific MP4 boxes in binary data
109
 * @param data - MP4 binary data
110
 * @param path - Array of box types to find (e.g., ['moov', 'trak'])
111
 * @returns Array of found boxes with offset and size
112
 */
113
function findBox(data: Uint8Array, path: string[]): BoxInfo[];
114

115
/**
116
 * Parse 4-character code box type
117
 * @param buffer - Data buffer
118
 * @param offset - Offset in buffer
119
 * @returns Parsed box type information
120
 */
121
function parseType(buffer: Uint8Array, offset: number): BoxType;
122

123
/**
124
 * Extract timescales from init segment
125
 * @param init - MP4 initialization segment
126
 * @returns Object mapping track IDs to timescales
127
 */
128
function timescale(init: Uint8Array): { [trackId: number]: number };
129

130
/**
131
 * Get fragment start time
132
 * @param timescale - Track timescale
133
 * @param fragment - MP4 fragment data
134
 * @returns Start time in timescale units
135
 */
136
function startTime(timescale: number, fragment: Uint8Array): number;
137

138
/**
139
 * Get composition start time from fragment
140
 * @param timescales - Track timescales object
141
 * @param fragment - MP4 fragment data  
142
 * @returns Composition start time
143
 */
144
function compositionStartTime(
145
  timescales: { [trackId: number]: number }, 
146
  fragment: Uint8Array
147
): number;
148

149
/**
150
 * Extract video track IDs from init segment
151
 * @param init - MP4 initialization segment
152
 * @returns Array of video track IDs
153
 */
154
function videoTrackIds(init: Uint8Array): number[];
155

156
/**
157
 * Get all track information from init segment
158
 * @param init - MP4 initialization segment
159
 * @returns Array of track information objects
160
 */
161
function tracks(init: Uint8Array): TrackInfo[];
162

163
/**
164
 * Parse EMSG box for ID3 metadata
165
 * @param segmentData - MP4 segment data
166
 * @param offset - Offset to EMSG box
167
 * @returns Parsed ID3 data
168
 */
169
function getEmsgID3(segmentData: Uint8Array, offset: number): ID3Data;
170

171
interface BoxInfo {
172
  type: string;
173
  size: number;
174
  offset: number;
175
  data: Uint8Array;
176
}
177

178
interface TrackInfo {
179
  id: number;
180
  type: 'audio' | 'video';
181
  timescale: number;
182
  codec: string;
183
}
184
```
185

186
**Usage Example:**
187

188
```javascript
189
const muxjs = require("mux.js");
190

191
// Analyze MP4 initialization segment
192
const tracks = muxjs.mp4.probe.tracks(initSegment);
193
console.log('Found tracks:', tracks);
194

195
const timescales = muxjs.mp4.probe.timescale(initSegment);
196
console.log('Track timescales:', timescales);
197

198
// Analyze fragment timing
199
const startTime = muxjs.mp4.probe.startTime(timescales[1], fragment);
200
console.log('Fragment start time:', startTime);
201

202
// Find specific boxes
203
const moovBoxes = muxjs.mp4.probe.findBox(mp4Data, ['moov']);
204
console.log('Found moov boxes:', moovBoxes);
205
```
206

207
### MP4 Box Generation
208

209
Low-level utilities for creating MP4 boxes and atoms programmatically.
210

211
```javascript { .api }
212
/**
213
 * Generate file type box (ftyp)
214
 * @returns ftyp box bytes
215
 */
216
function ftyp(): Uint8Array;
217

218
/**
219
 * Generate movie box (moov)
220
 * @param tracks - Array of track configurations
221
 * @param duration - Movie duration in timescale units
222
 * @returns moov box bytes
223
 */
224
function moov(tracks: TrackConfig[], duration: number): Uint8Array;
225

226
/**
227
 * Generate movie fragment box (moof)
228
 * @param sequenceNumber - Fragment sequence number
229
 * @param tracks - Array of track data
230
 * @returns moof box bytes
231
 */
232
function moof(sequenceNumber: number, tracks: TrackData[]): Uint8Array;
233

234
/**
235
 * Generate media data box (mdat)
236
 * @param data - Media data bytes
237
 * @returns mdat box bytes
238
 */
239
function mdat(data: Uint8Array): Uint8Array;
240

241
/**
242
 * Generate track box (trak) 
243
 * @param track - Track configuration
244
 * @returns trak box bytes
245
 */
246
function trak(track: TrackConfig): Uint8Array;
247

248
/**
249
 * Generate track run box (trun)
250
 * @param track - Track data with samples
251
 * @param offset - Data offset
252
 * @returns trun box bytes
253
 */
254
function trun(track: TrackData, offset: number): Uint8Array;
255

256
interface TrackConfig {
257
  id: number;
258
  type: 'audio' | 'video';
259
  timescale: number;
260
  duration: number;
261
  codec: string;
262
  // Additional track-specific properties
263
}
264

265
interface TrackData {
266
  id: number;
267
  samples: SampleData[];
268
}
269

270
interface SampleData {
271
  duration: number;
272
  size: number;
273
  flags: number;
274
  compositionTimeOffset?: number;
275
}
276
```
277

278
### Caption Parser
279

280
Specialized parser for extracting and processing caption data from MP4 containers.
281

282
```javascript { .api }
283
/**
284
 * Parser for caption data within MP4 containers
285
 */
286
class CaptionParser {
287
  constructor();
288
  
289
  /** Parse caption data from MP4 samples */
290
  parse(data: Uint8Array): CaptionData[];
291
  
292
  /** Reset parser state */
293
  reset(): void;
294
}
295

296
interface CaptionData {
297
  type: 'cea608' | 'cea708';
298
  pts: number;
299
  text: string;
300
  stream?: number;
301
}
302
```
303

304
## Advanced Usage
305

306
### Custom Transmuxing Pipeline
307

308
```javascript
309
const muxjs = require("mux.js");
310

311
// Create transmuxer with custom options
312
const transmuxer = new muxjs.mp4.Transmuxer({
313
  keepOriginalTimestamps: false,
314
  remux: true // Remove captions
315
});
316

317
// Set base decode time for timing alignment
318
transmuxer.setBaseMediaDecodeTime(90000); // 1 second at 90kHz
319

320
// Handle different segment types
321
transmuxer.on('data', function(segment) {
322
  if (segment.type === 'audio') {
323
    console.log('Audio segment - samples:', segment.boxes.mdat.audioSamples);
324
  } else if (segment.type === 'video') {
325
    console.log('Video segment - samples:', segment.boxes.mdat.videoSamples);
326
  }
327
  
328
  if (segment.captions && segment.captions.length > 0) {
329
    console.log('Captions found:', segment.captions);
330
  }
331
});
332
```
333

334
### Track Analysis and Metadata
335

336
```javascript
337
// Comprehensive track analysis
338
const initSegment = new Uint8Array(/* init segment data */);
339
const fragment = new Uint8Array(/* fragment data */);
340

341
// Get detailed track information
342
const tracks = muxjs.mp4.probe.tracks(initSegment);
343
tracks.forEach(track => {
344
  console.log(`Track ${track.id}:`);
345
  console.log(`  Type: ${track.type}`);
346
  console.log(`  Codec: ${track.codec}`);
347
  console.log(`  Timescale: ${track.timescale}`);
348
});
349

350
// Analyze timing across fragments
351
const timescales = muxjs.mp4.probe.timescale(initSegment);
352
const startTime = muxjs.mp4.probe.startTime(timescales[1], fragment);
353
const compositionTime = muxjs.mp4.probe.compositionStartTime(timescales, fragment);
354

355
console.log('Fragment timing:');
356
console.log(`  Start time: ${startTime}`);
357
console.log(`  Composition time: ${compositionTime}`);
358
```
359

360
## Error Handling
361

362
MP4 processing streams emit standard events for error handling:
363

364
```javascript
365
const transmuxer = new muxjs.mp4.Transmuxer();
366

367
transmuxer.on('error', function(error) {
368
  console.error('MP4 transmuxing error:', error.message);
369
  // Common errors: invalid TS data, unsupported codecs, timing issues
370
});
371

372
transmuxer.on('warning', function(warning) {
373
  console.warn('MP4 transmuxing warning:', warning.message);
374
  // Common warnings: missing frames, discontinuities, caption issues
375
});
376
```