0
# Media Streaming
1
2
Audio and video track management with abstract base classes for custom media sources and built-in track implementations for testing and development.
3
4
## Capabilities
5
6
### MediaStreamTrack Base Class
7
8
Abstract base class for all media tracks providing common interface and event handling.
9
10
```python { .api }
11
class MediaStreamTrack:
12
"""Abstract base class for media tracks."""
13
14
def __init__(self):
15
"""Initialize media track with unique ID and event emitter."""
16
17
@property
18
def id(self) -> str:
19
"""Unique track identifier"""
20
21
@property
22
def kind(self) -> str:
23
"""Media kind: "audio", "video", or "unknown" """
24
25
@property
26
def readyState(self) -> str:
27
"""Track state: "live" or "ended" """
28
29
async def recv(self):
30
"""
31
Receive next media frame or audio data.
32
33
This is an abstract method that must be implemented by subclasses.
34
35
Returns:
36
AudioFrame or VideoFrame: Next media data
37
"""
38
39
def stop(self) -> None:
40
"""Stop the track and emit 'ended' event."""
41
```
42
43
### AudioStreamTrack Class
44
45
Concrete audio track implementation that generates silence by default, suitable for testing or as a base class for custom audio sources.
46
47
```python { .api }
48
class AudioStreamTrack(MediaStreamTrack):
49
"""Audio track that generates silence by default."""
50
51
def __init__(self):
52
"""Initialize audio track."""
53
54
@property
55
def kind(self) -> str:
56
"""Always returns "audio" """
57
58
async def recv(self):
59
"""
60
Generate silent audio frame.
61
62
Returns:
63
AudioFrame: 20ms of silence at 8kHz mono
64
"""
65
```
66
67
### VideoStreamTrack Class
68
69
Concrete video track implementation that generates green frames by default, suitable for testing or as a base class for custom video sources.
70
71
```python { .api }
72
class VideoStreamTrack(MediaStreamTrack):
73
"""Video track that generates green frames by default."""
74
75
def __init__(self):
76
"""Initialize video track."""
77
78
@property
79
def kind(self) -> str:
80
"""Always returns "video" """
81
82
async def recv(self):
83
"""
84
Generate green video frame.
85
86
Returns:
87
VideoFrame: 640x480 green frame at 30fps
88
"""
89
90
async def next_timestamp(self):
91
"""
92
Get next timestamp for frame timing.
93
94
Returns:
95
tuple: (pts, time_base) for frame timing
96
"""
97
```
98
99
### Media Frame Types
100
101
Audio and video frame objects used by media tracks.
102
103
```python { .api }
104
class AudioFrame:
105
"""Audio frame containing PCM data."""
106
107
def __init__(self, format, layout, samples):
108
"""
109
Create audio frame.
110
111
Parameters:
112
- format: Audio format (e.g., "s16")
113
- layout: Channel layout (e.g., "mono", "stereo")
114
- samples: Number of samples per channel
115
"""
116
117
@property
118
def pts(self) -> int:
119
"""Presentation timestamp"""
120
121
@property
122
def time_base(self):
123
"""Time base for timestamp"""
124
125
class VideoFrame:
126
"""Video frame containing image data."""
127
128
def __init__(self, width, height, format="yuv420p"):
129
"""
130
Create video frame.
131
132
Parameters:
133
- width (int): Frame width in pixels
134
- height (int): Frame height in pixels
135
- format (str): Pixel format (default: "yuv420p")
136
"""
137
138
@property
139
def pts(self) -> int:
140
"""Presentation timestamp"""
141
142
@property
143
def time_base(self):
144
"""Time base for timestamp"""
145
146
@property
147
def width(self) -> int:
148
"""Frame width in pixels"""
149
150
@property
151
def height(self) -> int:
152
"""Frame height in pixels"""
153
```
154
155
### Media Constants
156
157
Timing and format constants for media processing.
158
159
```python { .api }
160
# Audio timing constants
161
AUDIO_PTIME = 0.020 # 20ms audio packet time
162
163
# Video timing constants
164
VIDEO_CLOCK_RATE = 90000 # 90kHz video clock
165
VIDEO_PTIME = 1/30 # 30fps frame time
166
```
167
168
### Media Stream Error
169
170
Exception raised when media stream operations fail.
171
172
```python { .api }
173
class MediaStreamError(Exception):
174
"""Exception for media stream errors."""
175
```
176
177
## Usage Examples
178
179
### Custom Audio Track
180
181
```python
182
import aiortc
183
import asyncio
184
import numpy as np
185
186
class SineWaveAudioTrack(aiortc.AudioStreamTrack):
187
"""Custom audio track generating sine wave."""
188
189
def __init__(self, frequency=440, sample_rate=8000):
190
super().__init__()
191
self.frequency = frequency
192
self.sample_rate = sample_rate
193
self.samples = 0
194
195
async def recv(self):
196
# Generate 20ms of sine wave audio
197
samples_per_frame = int(self.sample_rate * 0.020) # 20ms
198
t = np.arange(samples_per_frame) / self.sample_rate + self.samples / self.sample_rate
199
200
# Generate sine wave
201
audio_data = np.sin(2 * np.pi * self.frequency * t)
202
audio_data = (audio_data * 32767).astype(np.int16)
203
204
self.samples += samples_per_frame
205
206
# Create audio frame
207
frame = AudioFrame(format="s16", layout="mono", samples=samples_per_frame)
208
frame.planes[0].update(audio_data.tobytes())
209
210
# Set timing
211
pts, time_base = await self.next_timestamp()
212
frame.pts = pts
213
frame.time_base = time_base
214
215
return frame
216
217
# Use custom audio track
218
async def use_custom_audio():
219
pc = aiortc.RTCPeerConnection()
220
221
# Add custom sine wave audio track
222
audio_track = SineWaveAudioTrack(frequency=880) # A5 note
223
pc.addTrack(audio_track)
224
225
print(f"Added {audio_track.kind} track with ID: {audio_track.id}")
226
```
227
228
### Custom Video Track
229
230
```python
231
import aiortc
232
import cv2
233
import numpy as np
234
235
class WebcamVideoTrack(aiortc.VideoStreamTrack):
236
"""Custom video track from webcam."""
237
238
def __init__(self, device_id=0):
239
super().__init__()
240
self.cap = cv2.VideoCapture(device_id)
241
self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
242
self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
243
244
async def recv(self):
245
# Read frame from webcam
246
ret, frame = self.cap.read()
247
if not ret:
248
raise MediaStreamError("Failed to read from webcam")
249
250
# Convert BGR to RGB
251
frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
252
253
# Create video frame
254
video_frame = VideoFrame.from_ndarray(frame_rgb, format="rgb24")
255
256
# Set timing
257
pts, time_base = await self.next_timestamp()
258
video_frame.pts = pts
259
video_frame.time_base = time_base
260
261
return video_frame
262
263
def stop(self):
264
super().stop()
265
if self.cap:
266
self.cap.release()
267
268
# Use custom video track
269
async def use_custom_video():
270
pc = aiortc.RTCPeerConnection()
271
272
try:
273
# Add webcam video track
274
video_track = WebcamVideoTrack(device_id=0)
275
pc.addTrack(video_track)
276
277
print(f"Added {video_track.kind} track with ID: {video_track.id}")
278
279
# Process some frames
280
for i in range(10):
281
frame = await video_track.recv()
282
print(f"Frame {i}: {frame.width}x{frame.height}")
283
284
finally:
285
video_track.stop()
286
```
287
288
### Using Built-in Tracks
289
290
```python
291
async def use_builtin_tracks():
292
pc = aiortc.RTCPeerConnection()
293
294
# Add default tracks (silence and green frames)
295
audio_track = aiortc.AudioStreamTrack()
296
video_track = aiortc.VideoStreamTrack()
297
298
pc.addTrack(audio_track)
299
pc.addTrack(video_track)
300
301
print(f"Audio track ready state: {audio_track.readyState}")
302
print(f"Video track ready state: {video_track.readyState}")
303
304
# Test receiving frames
305
audio_frame = await audio_track.recv()
306
video_frame = await video_track.recv()
307
308
print(f"Audio frame: {audio_frame}")
309
print(f"Video frame: {video_frame.width}x{video_frame.height}")
310
311
# Stop tracks when done
312
audio_track.stop()
313
video_track.stop()
314
315
print(f"Audio track final state: {audio_track.readyState}")
316
print(f"Video track final state: {video_track.readyState}")
317
```
318
319
### Track Event Handling
320
321
```python
322
async def handle_track_events():
323
track = aiortc.VideoStreamTrack()
324
325
# Listen for track events
326
@track.on("ended")
327
def on_ended():
328
print("Track ended")
329
330
# Use track
331
print(f"Track state: {track.readyState}")
332
333
# Receive a few frames
334
for i in range(3):
335
frame = await track.recv()
336
print(f"Received frame {i+1}")
337
338
# Stop track (triggers 'ended' event)
339
track.stop()
340
```