0
# Video Rendering
1
2
Video rendering in ExoPlayer handles decoding, processing, and displaying video content. The video pipeline supports hardware acceleration, various surface outputs, and advanced features like spherical video and video effects.
3
4
## MediaCodecVideoRenderer
5
6
The primary video renderer that uses MediaCodec for hardware-accelerated video decoding.
7
8
```java { .api }
9
public class MediaCodecVideoRenderer extends MediaCodecRenderer {
10
/**
11
* Creates a MediaCodecVideoRenderer.
12
*
13
* @param context The context
14
* @param mediaCodecSelector The media codec selector
15
* @param allowedJoiningTimeMs The maximum time to block for joining
16
* @param eventHandler The event handler
17
* @param eventListener The event listener
18
* @param maxDroppedFramesToCountAsSkipped Max dropped frames to count as skipped
19
*/
20
public MediaCodecVideoRenderer(Context context, MediaCodecSelector mediaCodecSelector,
21
long allowedJoiningTimeMs, @Nullable Handler eventHandler,
22
@Nullable VideoRendererEventListener eventListener,
23
int maxDroppedFramesToCountAsSkipped);
24
25
/**
26
* Creates a MediaCodecVideoRenderer with a DrmSessionManager.
27
*
28
* @param context The context
29
* @param mediaCodecSelector The media codec selector
30
* @param allowedJoiningTimeMs The maximum time to block for joining
31
* @param drmSessionManager The DRM session manager
32
* @param playClearSamplesWithoutKeys Whether to play clear samples without keys
33
* @param eventHandler The event handler
34
* @param eventListener The event listener
35
* @param maxDroppedFramesToCountAsSkipped Max dropped frames to count as skipped
36
*/
37
public MediaCodecVideoRenderer(Context context, MediaCodecSelector mediaCodecSelector,
38
long allowedJoiningTimeMs, @Nullable DrmSessionManager drmSessionManager,
39
boolean playClearSamplesWithoutKeys, @Nullable Handler eventHandler,
40
@Nullable VideoRendererEventListener eventListener,
41
int maxDroppedFramesToCountAsSkipped);
42
43
/**
44
* Sets a video frame metadata listener.
45
*
46
* @param listener The listener
47
*/
48
public void setVideoFrameMetadataListener(VideoFrameMetadataListener listener);
49
50
/**
51
* Clears the video frame metadata listener.
52
*/
53
public void clearVideoFrameMetadataListener();
54
55
/**
56
* Sets the video surface.
57
*
58
* @param surface The surface
59
*/
60
public void setOutputSurface(@Nullable Surface surface);
61
62
/**
63
* Clears the video surface.
64
*/
65
public void clearOutputSurface();
66
}
67
```
68
69
## VideoSink Interface
70
71
Interface for consuming and rendering video frames.
72
73
```java { .api }
74
public interface VideoSink {
75
/**
76
* Exception thrown by video sink operations.
77
*/
78
final class VideoSinkException extends Exception {
79
public final Format format;
80
81
public VideoSinkException(Throwable cause, Format format);
82
}
83
84
/**
85
* Called when the input stream changes.
86
*
87
* @param inputType The input track type
88
* @param format The input format
89
*/
90
void onInputStreamChanged(@C.TrackType int inputType, Format format);
91
92
/**
93
* Registers an input stream.
94
*
95
* @param inputType The input track type
96
* @param effects The list of effects to apply
97
*/
98
void registerInputStream(@C.TrackType int inputType, List<Effect> effects);
99
100
/**
101
* Sets the output surface information.
102
*
103
* @param outputSurface The output surface
104
* @param outputResolution The output resolution
105
*/
106
void setOutputSurfaceInfo(@Nullable Surface outputSurface, @Nullable Size outputResolution);
107
108
/**
109
* Renders a video frame.
110
*
111
* @param presentationTimeUs The presentation time in microseconds
112
* @param releaseTimeNs The release time in nanoseconds
113
* @return Whether the frame was rendered successfully
114
* @throws VideoSinkException If rendering fails
115
*/
116
boolean render(long presentationTimeUs, long releaseTimeNs) throws VideoSinkException;
117
118
/**
119
* Joins the video sink, blocking until all queued frames are rendered.
120
*/
121
void join();
122
123
/**
124
* Releases the video sink.
125
*/
126
void release();
127
}
128
```
129
130
## VideoRendererEventListener
131
132
Listener for video renderer events.
133
134
```java { .api }
135
public interface VideoRendererEventListener {
136
/**
137
* Called when the video renderer is enabled.
138
*
139
* @param counters The decoder counters
140
*/
141
default void onVideoEnabled(DecoderCounters counters) {}
142
143
/**
144
* Called when the video decoder is initialized.
145
*
146
* @param decoderName The decoder name
147
* @param initializedTimestampMs The initialization timestamp
148
* @param initializationDurationMs The initialization duration
149
*/
150
default void onVideoDecoderInitialized(String decoderName, long initializedTimestampMs, long initializationDurationMs) {}
151
152
/**
153
* Called when the video input format changes.
154
*
155
* @param format The new format
156
* @param decoderReuseEvaluation The decoder reuse evaluation
157
*/
158
default void onVideoInputFormatChanged(Format format, @Nullable DecoderReuseEvaluation decoderReuseEvaluation) {}
159
160
/**
161
* Called when frames are dropped.
162
*
163
* @param count The number of dropped frames
164
* @param elapsed The elapsed time
165
*/
166
default void onDroppedVideoFrames(int count, long elapsed) {}
167
168
/**
169
* Called when the video size changes.
170
*
171
* @param videoSize The new video size
172
*/
173
default void onVideoSizeChanged(VideoSize videoSize) {}
174
175
/**
176
* Called when the first frame is rendered.
177
*
178
* @param output The output (Surface)
179
* @param renderTimeMs The render time
180
*/
181
default void onRenderedFirstFrame(Object output, long renderTimeMs) {}
182
183
/**
184
* Called when the video decoder is released.
185
*
186
* @param decoderName The decoder name
187
*/
188
default void onVideoDecoderReleased(String decoderName) {}
189
190
/**
191
* Called when the video renderer is disabled.
192
*
193
* @param counters The decoder counters
194
*/
195
default void onVideoDisabled(DecoderCounters counters) {}
196
197
/**
198
* Called when video frames are skipped.
199
*
200
* @param count The number of skipped frames
201
*/
202
default void onVideoFramesSkipped(int count) {}
203
204
/**
205
* Called when video codec error occurs.
206
*
207
* @param videoCodecError The video codec error
208
*/
209
default void onVideoCodecError(Exception videoCodecError) {}
210
}
211
```
212
213
## VideoFrameMetadataListener
214
215
Listener for video frame metadata.
216
217
```java { .api }
218
public interface VideoFrameMetadataListener {
219
/**
220
* Called when a video frame is about to be rendered.
221
*
222
* @param presentationTimeUs The presentation time in microseconds
223
* @param releaseTimeNs The release time in nanoseconds
224
* @param format The video format
225
* @param mediaFormat The media format
226
*/
227
void onVideoFrameAboutToBeRendered(long presentationTimeUs, long releaseTimeNs,
228
Format format, @Nullable MediaFormat mediaFormat);
229
}
230
```
231
232
## DecoderVideoRenderer
233
234
Base class for video renderers that use software decoders.
235
236
```java { .api }
237
public abstract class DecoderVideoRenderer extends BaseRenderer {
238
/**
239
* Creates a DecoderVideoRenderer.
240
*
241
* @param allowedJoiningTimeMs The maximum time to block for joining
242
* @param eventHandler The event handler
243
* @param eventListener The event listener
244
* @param maxDroppedFramesToCountAsSkipped Max dropped frames to count as skipped
245
*/
246
public DecoderVideoRenderer(long allowedJoiningTimeMs, @Nullable Handler eventHandler,
247
@Nullable VideoRendererEventListener eventListener,
248
int maxDroppedFramesToCountAsSkipped);
249
250
/**
251
* Sets the video surface.
252
*
253
* @param surface The surface
254
*/
255
public final void setOutputSurface(@Nullable Surface surface);
256
257
/**
258
* Clears the video surface.
259
*/
260
public final void clearOutputSurface();
261
262
/**
263
* Creates a decoder for the given format.
264
*
265
* @param format The format
266
* @param cryptoConfig The crypto config
267
* @return The decoder
268
* @throws DecoderException If decoder creation fails
269
*/
270
protected abstract VideoDecoder createDecoder(Format format, @Nullable CryptoConfig cryptoConfig) throws DecoderException;
271
272
/**
273
* Sets the decoder output mode.
274
*
275
* @param outputMode The output mode
276
*/
277
protected final void setDecoderOutputMode(@C.VideoOutputMode int outputMode);
278
}
279
```
280
281
## Spherical Video Support
282
283
Support for 360-degree and spherical video content.
284
285
```java { .api }
286
public final class SphericalGLSurfaceView extends GLSurfaceView {
287
/**
288
* Creates a SphericalGLSurfaceView.
289
*
290
* @param context The context
291
*/
292
public SphericalGLSurfaceView(Context context);
293
294
/**
295
* Creates a SphericalGLSurfaceView with attributes.
296
*
297
* @param context The context
298
* @param attrs The attributes
299
*/
300
public SphericalGLSurfaceView(Context context, AttributeSet attrs);
301
302
/**
303
* Sets the video component.
304
*
305
* @param player The ExoPlayer instance
306
*/
307
public void setVideoComponent(@Nullable VideoComponent videoComponent);
308
309
/**
310
* Sets whether single tap to hide UI is enabled.
311
*
312
* @param singleTapListener The tap listener, or null to disable
313
*/
314
public void setSingleTapListener(@Nullable SingleTapListener singleTapListener);
315
316
/**
317
* Sets the camera motion listener.
318
*
319
* @param listener The camera motion listener
320
*/
321
public void setCameraMotionListener(@Nullable CameraMotionListener listener);
322
323
/**
324
* Interface for handling single tap events.
325
*/
326
public interface SingleTapListener {
327
/**
328
* Called when the view is single tapped.
329
*
330
* @param e The motion event
331
* @return Whether the event was handled
332
*/
333
boolean onSingleTapUp(MotionEvent e);
334
}
335
}
336
337
public final class CameraMotionRenderer extends BaseRenderer {
338
/**
339
* Creates a CameraMotionRenderer.
340
*/
341
public CameraMotionRenderer();
342
343
/**
344
* Sets the camera motion listener.
345
*
346
* @param listener The listener
347
*/
348
public void setCameraMotionListener(@Nullable CameraMotionListener listener);
349
}
350
351
public interface CameraMotionListener {
352
/**
353
* Called when camera motion data is available.
354
*
355
* @param timeUs The timestamp in microseconds
356
* @param rotation The rotation matrix
357
*/
358
void onCameraMotion(long timeUs, float[] rotation);
359
360
/**
361
* Called when camera motion is reset.
362
*/
363
void onCameraMotionReset();
364
}
365
366
public interface ProjectionDecoder {
367
/**
368
* Decodes projection data from metadata.
369
*
370
* @param projectionData The projection data
371
* @param stereoMode The stereo mode
372
* @return The projection, or null if decoding fails
373
*/
374
@Nullable Projection decode(byte[] projectionData, @C.StereoMode int stereoMode);
375
}
376
```
377
378
## Usage Examples
379
380
### Basic Video Renderer Setup
381
382
```java
383
// Create video renderer with event listener
384
VideoRendererEventListener videoListener = new VideoRendererEventListener() {
385
@Override
386
public void onVideoSizeChanged(VideoSize videoSize) {
387
Log.d(TAG, "Video size: " + videoSize.width + "x" + videoSize.height);
388
// Update UI to match aspect ratio
389
updateVideoAspectRatio(videoSize);
390
}
391
392
@Override
393
public void onRenderedFirstFrame(Object output, long renderTimeMs) {
394
Log.d(TAG, "First frame rendered");
395
// Hide loading indicator
396
hideLoadingIndicator();
397
}
398
399
@Override
400
public void onDroppedVideoFrames(int count, long elapsed) {
401
Log.w(TAG, "Dropped " + count + " video frames in " + elapsed + "ms");
402
}
403
};
404
405
MediaCodecVideoRenderer videoRenderer = new MediaCodecVideoRenderer(
406
context,
407
MediaCodecSelector.DEFAULT,
408
5000, // 5 second joining time
409
handler,
410
videoListener,
411
50 // max dropped frames to count as skipped
412
);
413
```
414
415
### Surface Output Configuration
416
417
```java
418
// Set surface for video output
419
SurfaceView surfaceView = findViewById(R.id.surface_view);
420
Surface surface = surfaceView.getHolder().getSurface();
421
422
// Configure player with surface
423
PlayerView playerView = findViewById(R.id.player_view);
424
playerView.setPlayer(player);
425
426
// Or set surface directly on renderer
427
videoRenderer.setOutputSurface(surface);
428
429
// Update surface when size changes
430
surfaceView.getHolder().addCallback(new SurfaceHolder.Callback() {
431
@Override
432
public void surfaceCreated(SurfaceHolder holder) {
433
videoRenderer.setOutputSurface(holder.getSurface());
434
}
435
436
@Override
437
public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
438
// Surface size changed
439
}
440
441
@Override
442
public void surfaceDestroyed(SurfaceHolder holder) {
443
videoRenderer.clearOutputSurface();
444
}
445
});
446
```
447
448
### TextureView Output
449
450
```java
451
// Use TextureView for video output
452
TextureView textureView = findViewById(R.id.texture_view);
453
454
textureView.setSurfaceTextureListener(new TextureView.SurfaceTextureListener() {
455
@Override
456
public void onSurfaceTextureAvailable(SurfaceTexture surface, int width, int height) {
457
videoRenderer.setOutputSurface(new Surface(surface));
458
}
459
460
@Override
461
public void onSurfaceTextureSizeChanged(SurfaceTexture surface, int width, int height) {
462
// Handle size change
463
}
464
465
@Override
466
public boolean onSurfaceTextureDestroyed(SurfaceTexture surface) {
467
videoRenderer.clearOutputSurface();
468
return true;
469
}
470
471
@Override
472
public void onSurfaceTextureUpdated(SurfaceTexture surface) {
473
// Frame updated
474
}
475
});
476
```
477
478
### Video Frame Metadata
479
480
```java
481
VideoFrameMetadataListener frameMetadataListener = new VideoFrameMetadataListener() {
482
@Override
483
public void onVideoFrameAboutToBeRendered(long presentationTimeUs, long releaseTimeNs,
484
Format format, MediaFormat mediaFormat) {
485
// Log frame timing
486
Log.v(TAG, String.format("Frame at %dus, release at %dns",
487
presentationTimeUs, releaseTimeNs));
488
489
// Extract frame information
490
if (mediaFormat != null) {
491
int width = mediaFormat.getInteger(MediaFormat.KEY_WIDTH);
492
int height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT);
493
// Process frame metadata
494
}
495
}
496
};
497
498
// Set listener on renderer
499
((MediaCodecVideoRenderer) videoRenderer).setVideoFrameMetadataListener(frameMetadataListener);
500
```
501
502
### Spherical Video Playback
503
504
```java
505
// Setup spherical video view
506
SphericalGLSurfaceView sphericalView = findViewById(R.id.spherical_surface_view);
507
508
// Set player
509
sphericalView.setVideoComponent(player.getVideoComponent());
510
511
// Handle single tap to hide/show controls
512
sphericalView.setSingleTapListener(new SphericalGLSurfaceView.SingleTapListener() {
513
@Override
514
public boolean onSingleTapUp(MotionEvent e) {
515
toggleControlsVisibility();
516
return true;
517
}
518
});
519
520
// Set camera motion listener for gyroscope support
521
sphericalView.setCameraMotionListener(new CameraMotionListener() {
522
@Override
523
public void onCameraMotion(long timeUs, float[] rotation) {
524
// Handle camera motion data
525
updateViewOrientation(rotation);
526
}
527
528
@Override
529
public void onCameraMotionReset() {
530
// Reset view orientation
531
resetViewOrientation();
532
}
533
});
534
535
// Add camera motion renderer to player
536
CameraMotionRenderer cameraMotionRenderer = new CameraMotionRenderer();
537
cameraMotionRenderer.setCameraMotionListener(sphericalView.getCameraMotionListener());
538
539
// Use with custom renderers factory
540
RenderersFactory renderersFactory = new DefaultRenderersFactory(context) {
541
@Override
542
protected void buildCameraMotionRenderers(Context context, int extensionRendererMode,
543
ArrayList<Renderer> out) {
544
out.add(cameraMotionRenderer);
545
}
546
};
547
548
ExoPlayer player = new ExoPlayer.Builder(context)
549
.setRenderersFactory(renderersFactory)
550
.build();
551
```
552
553
### Video Effects
554
555
```java
556
// Apply video effects (requires Media3 Effects library)
557
List<Effect> videoEffects = Arrays.asList(
558
new ScaleAndRotateTransformation.Builder()
559
.setScale(1.5f, 1.5f) // Scale video
560
.setRotationDegrees(90) // Rotate 90 degrees
561
.build(),
562
new RgbFilter(/* red multiplier */ 1.2f, /* green */ 0.8f, /* blue */ 1.0f)
563
);
564
565
// Set effects on player
566
player.setVideoEffects(videoEffects);
567
```
568
569
### Custom Video Renderer
570
571
```java
572
public class CustomVideoRenderer extends DecoderVideoRenderer {
573
private VideoDecoder decoder;
574
575
public CustomVideoRenderer(long allowedJoiningTimeMs, Handler eventHandler,
576
VideoRendererEventListener eventListener,
577
int maxDroppedFramesToCountAsSkipped) {
578
super(allowedJoiningTimeMs, eventHandler, eventListener, maxDroppedFramesToCountAsSkipped);
579
}
580
581
@Override
582
protected VideoDecoder createDecoder(Format format, @Nullable CryptoConfig cryptoConfig)
583
throws DecoderException {
584
// Create custom decoder based on format
585
if (MimeTypes.VIDEO_H264.equals(format.sampleMimeType)) {
586
return new CustomH264Decoder();
587
} else if (MimeTypes.VIDEO_H265.equals(format.sampleMimeType)) {
588
return new CustomH265Decoder();
589
}
590
throw new DecoderException("Unsupported format: " + format.sampleMimeType);
591
}
592
593
@Override
594
public String getName() {
595
return "CustomVideoRenderer";
596
}
597
598
@Override
599
@Capabilities
600
public int supportsFormat(Format format) {
601
String mimeType = format.sampleMimeType;
602
if (MimeTypes.VIDEO_H264.equals(mimeType) || MimeTypes.VIDEO_H265.equals(mimeType)) {
603
return RendererCapabilities.create(C.FORMAT_HANDLED);
604
}
605
return RendererCapabilities.create(C.FORMAT_UNSUPPORTED_TYPE);
606
}
607
}
608
```
609
610
### Video Performance Monitoring
611
612
```java
613
public class VideoPerformanceMonitor implements VideoRendererEventListener {
614
private long firstFrameTime = -1;
615
private int totalDroppedFrames = 0;
616
private long sessionStartTime;
617
618
@Override
619
public void onVideoEnabled(DecoderCounters counters) {
620
sessionStartTime = SystemClock.elapsedRealtime();
621
firstFrameTime = -1;
622
totalDroppedFrames = 0;
623
Log.d(TAG, "Video renderer enabled");
624
}
625
626
@Override
627
public void onRenderedFirstFrame(Object output, long renderTimeMs) {
628
firstFrameTime = SystemClock.elapsedRealtime();
629
long timeToFirstFrame = firstFrameTime - sessionStartTime;
630
Log.i(TAG, "Time to first frame: " + timeToFirstFrame + "ms");
631
632
// Report to analytics
633
reportTimeToFirstFrame(timeToFirstFrame);
634
}
635
636
@Override
637
public void onDroppedVideoFrames(int count, long elapsed) {
638
totalDroppedFrames += count;
639
double dropRate = (double) count / (elapsed / 1000.0);
640
641
Log.w(TAG, String.format("Dropped %d frames in %dms (%.2f fps)",
642
count, elapsed, dropRate));
643
644
// Report significant frame drops
645
if (dropRate > 5.0) { // More than 5 fps drop rate
646
reportFrameDrop(count, elapsed, dropRate);
647
}
648
}
649
650
@Override
651
public void onVideoDisabled(DecoderCounters counters) {
652
long sessionDuration = SystemClock.elapsedRealtime() - sessionStartTime;
653
654
Log.i(TAG, String.format("Video session ended - Duration: %dms, Total dropped frames: %d",
655
sessionDuration, totalDroppedFrames));
656
657
reportSessionMetrics(sessionDuration, totalDroppedFrames, counters);
658
}
659
660
private void reportTimeToFirstFrame(long timeMs) {
661
// Send to analytics backend
662
}
663
664
private void reportFrameDrop(int count, long elapsed, double dropRate) {
665
// Send frame drop event to analytics
666
}
667
668
private void reportSessionMetrics(long duration, int droppedFrames, DecoderCounters counters) {
669
// Send overall session metrics
670
}
671
}
672
```