Source code

Revision control

Other Tools

1
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2
/* vim:set ts=2 sw=2 sts=2 et cindent: */
3
/* This Source Code Form is subject to the terms of the Mozilla Public
4
* License, v. 2.0. If a copy of the MPL was not distributed with this
5
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7
#include "FFmpegVideoDecoder.h"
8
#include "FFmpegLog.h"
9
#include "ImageContainer.h"
10
#include "MP4Decoder.h"
11
#include "MediaInfo.h"
12
#include "VPXDecoder.h"
13
#include "mozilla/layers/KnowsCompositor.h"
14
15
#include "libavutil/pixfmt.h"
16
#if LIBAVCODEC_VERSION_MAJOR < 54
17
# define AVPixelFormat PixelFormat
18
# define AV_PIX_FMT_YUV420P PIX_FMT_YUV420P
19
# define AV_PIX_FMT_YUVJ420P PIX_FMT_YUVJ420P
20
# define AV_PIX_FMT_YUV420P10LE PIX_FMT_YUV420P10LE
21
# define AV_PIX_FMT_YUV422P PIX_FMT_YUV422P
22
# define AV_PIX_FMT_YUV422P10LE PIX_FMT_YUV422P10LE
23
# define AV_PIX_FMT_YUV444P PIX_FMT_YUV444P
24
# define AV_PIX_FMT_YUV444P10LE PIX_FMT_YUV444P10LE
25
# define AV_PIX_FMT_NONE PIX_FMT_NONE
26
#endif
27
#include "mozilla/PodOperations.h"
28
#include "mozilla/TaskQueue.h"
29
#include "nsThreadUtils.h"
30
#include "prsystem.h"
31
32
typedef mozilla::layers::Image Image;
33
typedef mozilla::layers::PlanarYCbCrImage PlanarYCbCrImage;
34
35
namespace mozilla {
36
37
using media::TimeUnit;
38
39
/**
40
* FFmpeg calls back to this function with a list of pixel formats it supports.
41
* We choose a pixel format that we support and return it.
42
* For now, we just look for YUV420P, YUVJ420P and YUV444 as those are the only
43
* only non-HW accelerated format supported by FFmpeg's H264 and VP9 decoder.
44
*/
45
static AVPixelFormat ChoosePixelFormat(AVCodecContext* aCodecContext,
46
const AVPixelFormat* aFormats) {
47
FFMPEG_LOG("Choosing FFmpeg pixel format for video decoding.");
48
for (; *aFormats > -1; aFormats++) {
49
switch (*aFormats) {
50
case AV_PIX_FMT_YUV420P:
51
FFMPEG_LOG("Requesting pixel format YUV420P.");
52
return AV_PIX_FMT_YUV420P;
53
case AV_PIX_FMT_YUVJ420P:
54
FFMPEG_LOG("Requesting pixel format YUVJ420P.");
55
return AV_PIX_FMT_YUVJ420P;
56
case AV_PIX_FMT_YUV420P10LE:
57
FFMPEG_LOG("Requesting pixel format YUV420P10LE.");
58
return AV_PIX_FMT_YUV420P10LE;
59
case AV_PIX_FMT_YUV422P:
60
FFMPEG_LOG("Requesting pixel format YUV422P.");
61
return AV_PIX_FMT_YUV422P;
62
case AV_PIX_FMT_YUV422P10LE:
63
FFMPEG_LOG("Requesting pixel format YUV422P10LE.");
64
return AV_PIX_FMT_YUV422P10LE;
65
case AV_PIX_FMT_YUV444P:
66
FFMPEG_LOG("Requesting pixel format YUV444P.");
67
return AV_PIX_FMT_YUV444P;
68
case AV_PIX_FMT_YUV444P10LE:
69
FFMPEG_LOG("Requesting pixel format YUV444P10LE.");
70
return AV_PIX_FMT_YUV444P10LE;
71
#if LIBAVCODEC_VERSION_MAJOR >= 57
72
case AV_PIX_FMT_YUV420P12LE:
73
FFMPEG_LOG("Requesting pixel format YUV420P12LE.");
74
return AV_PIX_FMT_YUV420P12LE;
75
case AV_PIX_FMT_YUV422P12LE:
76
FFMPEG_LOG("Requesting pixel format YUV422P12LE.");
77
return AV_PIX_FMT_YUV422P12LE;
78
case AV_PIX_FMT_YUV444P12LE:
79
FFMPEG_LOG("Requesting pixel format YUV444P12LE.");
80
return AV_PIX_FMT_YUV444P12LE;
81
#endif
82
default:
83
break;
84
}
85
}
86
87
NS_WARNING("FFmpeg does not share any supported pixel formats.");
88
return AV_PIX_FMT_NONE;
89
}
90
91
FFmpegVideoDecoder<LIBAV_VER>::PtsCorrectionContext::PtsCorrectionContext()
92
: mNumFaultyPts(0),
93
mNumFaultyDts(0),
94
mLastPts(INT64_MIN),
95
mLastDts(INT64_MIN) {}
96
97
int64_t FFmpegVideoDecoder<LIBAV_VER>::PtsCorrectionContext::GuessCorrectPts(
98
int64_t aPts, int64_t aDts) {
99
int64_t pts = AV_NOPTS_VALUE;
100
101
if (aDts != int64_t(AV_NOPTS_VALUE)) {
102
mNumFaultyDts += aDts <= mLastDts;
103
mLastDts = aDts;
104
}
105
if (aPts != int64_t(AV_NOPTS_VALUE)) {
106
mNumFaultyPts += aPts <= mLastPts;
107
mLastPts = aPts;
108
}
109
if ((mNumFaultyPts <= mNumFaultyDts || aDts == int64_t(AV_NOPTS_VALUE)) &&
110
aPts != int64_t(AV_NOPTS_VALUE)) {
111
pts = aPts;
112
} else {
113
pts = aDts;
114
}
115
return pts;
116
}
117
118
void FFmpegVideoDecoder<LIBAV_VER>::PtsCorrectionContext::Reset() {
119
mNumFaultyPts = 0;
120
mNumFaultyDts = 0;
121
mLastPts = INT64_MIN;
122
mLastDts = INT64_MIN;
123
}
124
125
FFmpegVideoDecoder<LIBAV_VER>::FFmpegVideoDecoder(
126
FFmpegLibWrapper* aLib, TaskQueue* aTaskQueue, const VideoInfo& aConfig,
127
KnowsCompositor* aAllocator, ImageContainer* aImageContainer,
128
bool aLowLatency)
129
: FFmpegDataDecoder(aLib, aTaskQueue, GetCodecId(aConfig.mMimeType)),
130
mImageAllocator(aAllocator),
131
mImageContainer(aImageContainer),
132
mInfo(aConfig),
133
mLowLatency(aLowLatency) {
134
// Use a new MediaByteBuffer as the object will be modified during
135
// initialization.
136
mExtraData = new MediaByteBuffer;
137
mExtraData->AppendElements(*aConfig.mExtraData);
138
}
139
140
RefPtr<MediaDataDecoder::InitPromise> FFmpegVideoDecoder<LIBAV_VER>::Init() {
141
MediaResult rv = InitDecoder();
142
if (NS_FAILED(rv)) {
143
return InitPromise::CreateAndReject(rv, __func__);
144
}
145
146
return InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, __func__);
147
}
148
149
void FFmpegVideoDecoder<LIBAV_VER>::InitCodecContext() {
150
mCodecContext->width = mInfo.mImage.width;
151
mCodecContext->height = mInfo.mImage.height;
152
153
// We use the same logic as libvpx in determining the number of threads to use
154
// so that we end up behaving in the same fashion when using ffmpeg as
155
// we would otherwise cause various crashes (see bug 1236167)
156
int decode_threads = 1;
157
if (mInfo.mDisplay.width >= 2048) {
158
decode_threads = 8;
159
} else if (mInfo.mDisplay.width >= 1024) {
160
decode_threads = 4;
161
} else if (mInfo.mDisplay.width >= 320) {
162
decode_threads = 2;
163
}
164
165
if (mLowLatency) {
166
mCodecContext->flags |= AV_CODEC_FLAG_LOW_DELAY;
167
// ffvp9 and ffvp8 at this stage do not support slice threading, but it may
168
// help with the h264 decoder if there's ever one.
169
mCodecContext->thread_type = FF_THREAD_SLICE;
170
} else {
171
decode_threads = std::min(decode_threads, PR_GetNumberOfProcessors() - 1);
172
decode_threads = std::max(decode_threads, 1);
173
mCodecContext->thread_count = decode_threads;
174
if (decode_threads > 1) {
175
mCodecContext->thread_type = FF_THREAD_SLICE | FF_THREAD_FRAME;
176
}
177
}
178
179
// FFmpeg will call back to this to negotiate a video pixel format.
180
mCodecContext->get_format = ChoosePixelFormat;
181
}
182
183
MediaResult FFmpegVideoDecoder<LIBAV_VER>::DoDecode(
184
MediaRawData* aSample, uint8_t* aData, int aSize, bool* aGotFrame,
185
MediaDataDecoder::DecodedData& aResults) {
186
AVPacket packet;
187
mLib->av_init_packet(&packet);
188
189
packet.data = aData;
190
packet.size = aSize;
191
packet.dts = aSample->mTimecode.ToMicroseconds();
192
packet.pts = aSample->mTime.ToMicroseconds();
193
packet.flags = aSample->mKeyframe ? AV_PKT_FLAG_KEY : 0;
194
packet.pos = aSample->mOffset;
195
196
#if LIBAVCODEC_VERSION_MAJOR >= 58
197
packet.duration = aSample->mDuration.ToMicroseconds();
198
int res = mLib->avcodec_send_packet(mCodecContext, &packet);
199
if (res < 0) {
200
// In theory, avcodec_send_packet could sent -EAGAIN should its internal
201
// buffers be full. In practice this can't happen as we only feed one frame
202
// at a time, and we immediately call avcodec_receive_frame right after.
203
FFMPEG_LOG("avcodec_send_packet error: %d", res);
204
return MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
205
RESULT_DETAIL("avcodec_send_packet error: %d", res));
206
}
207
208
if (aGotFrame) {
209
*aGotFrame = false;
210
}
211
do {
212
if (!PrepareFrame()) {
213
NS_WARNING("FFmpeg h264 decoder failed to allocate frame.");
214
return MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__);
215
}
216
res = mLib->avcodec_receive_frame(mCodecContext, mFrame);
217
if (res == int(AVERROR_EOF)) {
218
return NS_ERROR_DOM_MEDIA_END_OF_STREAM;
219
}
220
if (res == AVERROR(EAGAIN)) {
221
return NS_OK;
222
}
223
if (res < 0) {
224
FFMPEG_LOG("avcodec_receive_frame error: %d", res);
225
return MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
226
RESULT_DETAIL("avcodec_receive_frame error: %d", res));
227
}
228
MediaResult rv = CreateImage(mFrame->pkt_pos, mFrame->pkt_pts,
229
mFrame->pkt_duration, aResults);
230
if (NS_FAILED(rv)) {
231
return rv;
232
}
233
if (aGotFrame) {
234
*aGotFrame = true;
235
}
236
} while (true);
237
#else
238
// LibAV provides no API to retrieve the decoded sample's duration.
239
// (FFmpeg >= 1.0 provides av_frame_get_pkt_duration)
240
// As such we instead use a map using the dts as key that we will retrieve
241
// later.
242
// The map will have a typical size of 16 entry.
243
mDurationMap.Insert(aSample->mTimecode.ToMicroseconds(),
244
aSample->mDuration.ToMicroseconds());
245
246
if (!PrepareFrame()) {
247
NS_WARNING("FFmpeg h264 decoder failed to allocate frame.");
248
return MediaResult(NS_ERROR_OUT_OF_MEMORY, __func__);
249
}
250
251
// Required with old version of FFmpeg/LibAV
252
mFrame->reordered_opaque = AV_NOPTS_VALUE;
253
254
int decoded;
255
int bytesConsumed =
256
mLib->avcodec_decode_video2(mCodecContext, mFrame, &decoded, &packet);
257
258
FFMPEG_LOG(
259
"DoDecodeFrame:decode_video: rv=%d decoded=%d "
260
"(Input: pts(%" PRId64 ") dts(%" PRId64 ") Output: pts(%" PRId64
261
") "
262
"opaque(%" PRId64 ") pkt_pts(%" PRId64 ") pkt_dts(%" PRId64 "))",
263
bytesConsumed, decoded, packet.pts, packet.dts, mFrame->pts,
264
mFrame->reordered_opaque, mFrame->pkt_pts, mFrame->pkt_dts);
265
266
if (bytesConsumed < 0) {
267
return MediaResult(NS_ERROR_DOM_MEDIA_DECODE_ERR,
268
RESULT_DETAIL("FFmpeg video error:%d", bytesConsumed));
269
}
270
271
if (!decoded) {
272
if (aGotFrame) {
273
*aGotFrame = false;
274
}
275
return NS_OK;
276
}
277
278
// If we've decoded a frame then we need to output it
279
int64_t pts = mPtsContext.GuessCorrectPts(mFrame->pkt_pts, mFrame->pkt_dts);
280
// Retrieve duration from dts.
281
// We use the first entry found matching this dts (this is done to
282
// handle damaged file with multiple frames with the same dts)
283
284
int64_t duration;
285
if (!mDurationMap.Find(mFrame->pkt_dts, duration)) {
286
NS_WARNING("Unable to retrieve duration from map");
287
duration = aSample->mDuration.ToMicroseconds();
288
// dts are probably incorrectly reported ; so clear the map as we're
289
// unlikely to find them in the future anyway. This also guards
290
// against the map becoming extremely big.
291
mDurationMap.Clear();
292
}
293
294
MediaResult rv = CreateImage(aSample->mOffset, pts, duration, aResults);
295
if (NS_SUCCEEDED(rv) && aGotFrame) {
296
*aGotFrame = true;
297
}
298
return rv;
299
#endif
300
}
301
302
MediaResult FFmpegVideoDecoder<LIBAV_VER>::CreateImage(
303
int64_t aOffset, int64_t aPts, int64_t aDuration,
304
MediaDataDecoder::DecodedData& aResults) {
305
FFMPEG_LOG("Got one frame output with pts=%" PRId64 " dts=%" PRId64
306
" duration=%" PRId64 " opaque=%" PRId64,
307
aPts, mFrame->pkt_dts, aDuration, mCodecContext->reordered_opaque);
308
309
VideoData::YCbCrBuffer b;
310
b.mPlanes[0].mData = mFrame->data[0];
311
b.mPlanes[1].mData = mFrame->data[1];
312
b.mPlanes[2].mData = mFrame->data[2];
313
314
b.mPlanes[0].mStride = mFrame->linesize[0];
315
b.mPlanes[1].mStride = mFrame->linesize[1];
316
b.mPlanes[2].mStride = mFrame->linesize[2];
317
318
b.mPlanes[0].mOffset = b.mPlanes[0].mSkip = 0;
319
b.mPlanes[1].mOffset = b.mPlanes[1].mSkip = 0;
320
b.mPlanes[2].mOffset = b.mPlanes[2].mSkip = 0;
321
322
b.mPlanes[0].mWidth = mFrame->width;
323
b.mPlanes[0].mHeight = mFrame->height;
324
if (mCodecContext->pix_fmt == AV_PIX_FMT_YUV444P ||
325
mCodecContext->pix_fmt == AV_PIX_FMT_YUV444P10LE
326
#if LIBAVCODEC_VERSION_MAJOR >= 57
327
|| mCodecContext->pix_fmt == AV_PIX_FMT_YUV444P12LE
328
#endif
329
) {
330
b.mPlanes[1].mWidth = b.mPlanes[2].mWidth = mFrame->width;
331
b.mPlanes[1].mHeight = b.mPlanes[2].mHeight = mFrame->height;
332
if (mCodecContext->pix_fmt == AV_PIX_FMT_YUV444P10LE) {
333
b.mColorDepth = gfx::ColorDepth::COLOR_10;
334
}
335
#if LIBAVCODEC_VERSION_MAJOR >= 57
336
else if (mCodecContext->pix_fmt == AV_PIX_FMT_YUV444P12LE) {
337
b.mColorDepth = gfx::ColorDepth::COLOR_12;
338
}
339
#endif
340
} else if (mCodecContext->pix_fmt == AV_PIX_FMT_YUV422P ||
341
mCodecContext->pix_fmt == AV_PIX_FMT_YUV422P10LE
342
#if LIBAVCODEC_VERSION_MAJOR >= 57
343
|| mCodecContext->pix_fmt == AV_PIX_FMT_YUV422P12LE
344
#endif
345
) {
346
b.mPlanes[1].mWidth = b.mPlanes[2].mWidth = (mFrame->width + 1) >> 1;
347
b.mPlanes[1].mHeight = b.mPlanes[2].mHeight = mFrame->height;
348
if (mCodecContext->pix_fmt == AV_PIX_FMT_YUV422P10LE) {
349
b.mColorDepth = gfx::ColorDepth::COLOR_10;
350
}
351
#if LIBAVCODEC_VERSION_MAJOR >= 57
352
else if (mCodecContext->pix_fmt == AV_PIX_FMT_YUV422P12LE) {
353
b.mColorDepth = gfx::ColorDepth::COLOR_12;
354
}
355
#endif
356
} else {
357
b.mPlanes[1].mWidth = b.mPlanes[2].mWidth = (mFrame->width + 1) >> 1;
358
b.mPlanes[1].mHeight = b.mPlanes[2].mHeight = (mFrame->height + 1) >> 1;
359
if (mCodecContext->pix_fmt == AV_PIX_FMT_YUV420P10LE) {
360
b.mColorDepth = gfx::ColorDepth::COLOR_10;
361
}
362
#if LIBAVCODEC_VERSION_MAJOR >= 57
363
else if (mCodecContext->pix_fmt == AV_PIX_FMT_YUV420P12LE) {
364
b.mColorDepth = gfx::ColorDepth::COLOR_12;
365
}
366
#endif
367
}
368
if (mLib->av_frame_get_colorspace) {
369
switch (mLib->av_frame_get_colorspace(mFrame)) {
370
#if LIBAVCODEC_VERSION_MAJOR >= 55
371
case AVCOL_SPC_BT2020_NCL:
372
case AVCOL_SPC_BT2020_CL:
373
b.mYUVColorSpace = gfx::YUVColorSpace::BT2020;
374
break;
375
#endif
376
case AVCOL_SPC_BT709:
377
b.mYUVColorSpace = gfx::YUVColorSpace::BT709;
378
break;
379
case AVCOL_SPC_SMPTE170M:
380
case AVCOL_SPC_BT470BG:
381
b.mYUVColorSpace = gfx::YUVColorSpace::BT601;
382
break;
383
case AVCOL_SPC_UNSPECIFIED:
384
b.mYUVColorSpace = gfx::YUVColorSpace::UNKNOWN;
385
break;
386
default:
387
break;
388
}
389
}
390
if (b.mYUVColorSpace == gfx::YUVColorSpace::UNKNOWN) {
391
b.mYUVColorSpace = DefaultColorSpace({mFrame->width, mFrame->height});
392
}
393
394
if (mLib->av_frame_get_color_range) {
395
auto range = mLib->av_frame_get_color_range(mFrame);
396
b.mColorRange = range == AVCOL_RANGE_JPEG ? gfx::ColorRange::FULL
397
: gfx::ColorRange::LIMITED;
398
}
399
400
RefPtr<VideoData> v = VideoData::CreateAndCopyData(
401
mInfo, mImageContainer, aOffset, TimeUnit::FromMicroseconds(aPts),
402
TimeUnit::FromMicroseconds(aDuration), b, !!mFrame->key_frame,
403
TimeUnit::FromMicroseconds(-1),
404
mInfo.ScaledImageRect(mFrame->width, mFrame->height), mImageAllocator);
405
406
if (!v) {
407
return MediaResult(NS_ERROR_OUT_OF_MEMORY,
408
RESULT_DETAIL("image allocation error"));
409
}
410
aResults.AppendElement(std::move(v));
411
return NS_OK;
412
}
413
414
RefPtr<MediaDataDecoder::FlushPromise>
415
FFmpegVideoDecoder<LIBAV_VER>::ProcessFlush() {
416
mPtsContext.Reset();
417
mDurationMap.Clear();
418
return FFmpegDataDecoder::ProcessFlush();
419
}
420
421
AVCodecID FFmpegVideoDecoder<LIBAV_VER>::GetCodecId(
422
const nsACString& aMimeType) {
423
if (MP4Decoder::IsH264(aMimeType)) {
424
return AV_CODEC_ID_H264;
425
}
426
427
if (aMimeType.EqualsLiteral("video/x-vnd.on2.vp6")) {
428
return AV_CODEC_ID_VP6F;
429
}
430
431
#if LIBAVCODEC_VERSION_MAJOR >= 54
432
if (VPXDecoder::IsVP8(aMimeType)) {
433
return AV_CODEC_ID_VP8;
434
}
435
#endif
436
437
#if LIBAVCODEC_VERSION_MAJOR >= 55
438
if (VPXDecoder::IsVP9(aMimeType)) {
439
return AV_CODEC_ID_VP9;
440
}
441
#endif
442
443
return AV_CODEC_ID_NONE;
444
}
445
446
} // namespace mozilla