0
# Codec Processing
1

2
Specialized codec parsers for H.264 video and ADTS/AAC audio streams. The codecs module handles bitstream parsing, NAL unit extraction, and audio frame processing within the mux.js streaming pipeline.
3

4
## Capabilities
5

6
### ADTS Audio Processing
7

8
Parser for ADTS (Audio Data Transport Stream) format that extracts AAC audio frames from transport streams.
9

10
```javascript { .api }
11
/**
12
 * ADTS stream parser for AAC audio processing
13
 * Accepts ElementaryStream input and emits parsed AAC audio frames
14
 */
15
class Adts {
16
  constructor();
17
  
18
  /** Process ADTS data and emit AAC frames */
19
  push(data: Uint8Array): void;
20
  
21
  /** Flush any remaining data */
22
  flush(): void;
23
  
24
  /** Reset parser state */
25
  reset(): void;
26
  
27
  /** Register event listeners */
28
  on(event: 'data', callback: (frame: AACFrame) => void): void;
29
  on(event: 'done', callback: () => void): void;
30
}
31

32
interface AACFrame {
33
  data: Uint8Array;
34
  pts: number;
35
  dts: number;
36
  sampleRate: number;
37
  channelCount: number;
38
  sampleCount: number;
39
}
40
```
41

42
**Usage Examples:**
43

44
```javascript
45
const muxjs = require("mux.js");
46

47
// Create ADTS parser
48
const adtsParser = new muxjs.codecs.Adts();
49

50
// Handle parsed AAC frames
51
adtsParser.on('data', (frame) => {
52
  console.log(`AAC frame: ${frame.data.length} bytes`);
53
  console.log(`Sample rate: ${frame.sampleRate}Hz`);
54
  console.log(`Channels: ${frame.channelCount}`);
55
  console.log(`PTS: ${frame.pts}`);
56
  
57
  // Process AAC frame data
58
  processAACFrame(frame);
59
});
60

61
// Process ADTS data from elementary stream
62
adtsParser.push(adtsData);
63
adtsParser.flush();
64
```
65

66
### H.264 Video Processing
67

68
Comprehensive H.264 bitstream processing including NAL unit parsing and video frame extraction.
69

70
```javascript { .api }
71
const h264: {
72
  /** H.264 stream parser for video processing */
73
  H264Stream: class H264Stream {
74
    constructor();
75
    
76
    /** Process H.264 data and emit video frames */
77
    push(data: Uint8Array): void;
78
    
79
    /** Flush any remaining data */
80
    flush(): void;
81
    
82
    /** Reset parser state */
83
    reset(): void;
84
    
85
    /** Register event listeners */
86
    on(event: 'data', callback: (frame: H264Frame) => void): void;
87
  };
88
  
89
  /** NAL unit byte stream parser */
90
  NalByteStream: class NalByteStream {
91
    constructor();
92
    
93
    /** Process NAL byte stream and emit NAL units */
94
    push(data: Uint8Array): void;
95
    
96
    /** Flush any remaining data */
97
    flush(): void;
98
    
99
    /** Register event listeners */
100
    on(event: 'data', callback: (nalUnit: NALUnit) => void): void;
101
  };
102
};
103

104
interface H264Frame {
105
  data: Uint8Array;
106
  pts: number;
107
  dts: number;
108
  keyframe: boolean;
109
  nalUnits: NALUnit[];
110
}
111

112
interface NALUnit {
113
  data: Uint8Array;
114
  type: number;
115
  nalUnitType: string;
116
}
117
```
118

119
**Usage Examples:**
120

121
```javascript
122
const muxjs = require("mux.js");
123

124
// Create H.264 stream parser
125
const h264Stream = new muxjs.codecs.h264.H264Stream();
126

127
// Handle parsed video frames
128
h264Stream.on('data', (frame) => {
129
  console.log(`H.264 frame: ${frame.data.length} bytes`);
130
  console.log(`Keyframe: ${frame.keyframe}`);
131
  console.log(`NAL units: ${frame.nalUnits.length}`);
132
  console.log(`PTS: ${frame.pts}, DTS: ${frame.dts}`);
133
  
134
  // Process video frame
135
  processVideoFrame(frame);
136
});
137

138
// Create NAL byte stream parser
139
const nalStream = new muxjs.codecs.h264.NalByteStream();
140

141
// Handle parsed NAL units
142
nalStream.on('data', (nalUnit) => {
143
  console.log(`NAL unit type: ${nalUnit.nalUnitType} (${nalUnit.type})`);
144
  console.log(`Data length: ${nalUnit.data.length} bytes`);
145
  
146
  // Process NAL unit
147
  processNALUnit(nalUnit);
148
});
149

150
// Chain parsers together
151
nalStream.pipe(h264Stream);
152

153
// Process H.264 bitstream data
154
nalStream.push(h264BitstreamData);
155
nalStream.flush();
156
```
157

158
### Codec Pipeline Integration
159

160
The codec parsers integrate seamlessly with the mux.js streaming pipeline architecture.
161

162
```javascript { .api }
163
/**
164
 * Base Stream class that all codec parsers extend
165
 */
166
class Stream {
167
  /** Connect this stream to another stream */
168
  pipe(destination: Stream): Stream;
169
  
170
  /** Register event listener */
171
  on(event: string, callback: Function): void;
172
  
173
  /** Remove event listener */
174
  off(event?: string): void;
175
  
176
  /** Trigger event */
177
  trigger(event: string, ...args: any[]): void;
178
}
179
```
180

181
**Usage Examples:**
182

183
```javascript
184
const muxjs = require("mux.js");
185

186
// Create processing pipeline
187
const transportPacketStream = new muxjs.mp2t.TransportPacketStream();
188
const transportParseStream = new muxjs.mp2t.TransportParseStream();
189
const elementaryStream = new muxjs.mp2t.ElementaryStream();
190
const adtsParser = new muxjs.codecs.Adts();
191
const h264Parser = new muxjs.codecs.h264.H264Stream();
192

193
// Connect pipeline for audio processing
194
transportPacketStream
195
  .pipe(transportParseStream)
196
  .pipe(elementaryStream);
197

198
// Handle elementary stream data
199
elementaryStream.on('data', (data) => {
200
  if (data.streamType === muxjs.mp2t.ADTS_STREAM_TYPE) {
201
    adtsParser.push(data.data);
202
  } else if (data.streamType === muxjs.mp2t.H264_STREAM_TYPE) {
203
    h264Parser.push(data.data);
204
  }
205
});
206

207
// Process transport stream
208
transportPacketStream.push(transportStreamData);
209
transportPacketStream.flush();
210
```
211

212
## Advanced Codec Features
213

214
### AAC Configuration Detection
215

216
The ADTS parser automatically detects AAC configuration from the bitstream.
217

218
```javascript
219
// ADTS parser automatically extracts audio configuration
220
adtsParser.on('data', (frame) => {
221
  // Audio configuration is available in frame properties
222
  const config = {
223
    sampleRate: frame.sampleRate,    // e.g., 44100, 48000
224
    channelCount: frame.channelCount, // e.g., 1, 2, 6
225
    profile: frame.profile,          // AAC profile
226
    sampleCount: frame.sampleCount   // Samples per frame (typically 1024)
227
  };
228
  
229
  console.log('AAC Configuration:', config);
230
});
231
```
232

233
### H.264 NAL Unit Types
234

235
The H.264 parser identifies different NAL unit types for proper processing.
236

237
```javascript
238
// NAL unit types commonly encountered
239
const NAL_UNIT_TYPES = {
240
  1: 'NON_IDR_PICTURE',
241
  5: 'IDR_PICTURE', 
242
  6: 'SEI',
243
  7: 'SPS',  // Sequence Parameter Set
244
  8: 'PPS',  // Picture Parameter Set
245
  9: 'ACCESS_UNIT_DELIMITER'
246
};
247

248
nalStream.on('data', (nalUnit) => {
249
  const typeName = NAL_UNIT_TYPES[nalUnit.type] || 'UNKNOWN';
250
  console.log(`NAL Unit: ${typeName} (${nalUnit.type})`);
251
  
252
  // Handle specific NAL unit types
253
  switch (nalUnit.type) {
254
    case 7: // SPS
255
      console.log('Found Sequence Parameter Set');
256
      break;
257
    case 8: // PPS  
258
      console.log('Found Picture Parameter Set');
259
      break;
260
    case 5: // IDR
261
      console.log('Found keyframe (IDR)');
262
      break;
263
  }
264
});
265
```
266

267
## Types
268

269
```javascript { .api }
270
interface CodecFrame {
271
  data: Uint8Array;
272
  pts: number;
273
  dts: number;
274
}
275

276
interface AACFrame extends CodecFrame {
277
  sampleRate: number;
278
  channelCount: number;
279
  sampleCount: number;
280
  profile?: number;
281
}
282

283
interface H264Frame extends CodecFrame {
284
  keyframe: boolean;
285
  nalUnits: NALUnit[];
286
}
287

288
interface NALUnit {
289
  data: Uint8Array;
290
  type: number;
291
  nalUnitType: string;
292
}
293

294
interface StreamEvent {
295
  type: 'video' | 'audio';
296
  data: Uint8Array;
297
  pts?: number;
298
  dts?: number;
299
  streamType: number;
300
}
301
```