Source code

Revision control

Copy as Markdown

Other Tools

/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set ts=2 sw=2 sts=2 et cindent: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "FFmpegVideoDecoder.h"
#include "FFmpegLog.h"
#include "FFmpegUtils.h"
#include "ImageContainer.h"
#include "MP4Decoder.h"
#include "MediaInfo.h"
#include "VideoUtils.h"
#include "VPXDecoder.h"
#include "mozilla/layers/KnowsCompositor.h"
#include "nsPrintfCString.h"
#if LIBAVCODEC_VERSION_MAJOR >= 57
# include "mozilla/layers/TextureClient.h"
#endif
#if LIBAVCODEC_VERSION_MAJOR >= 58
# include "mozilla/ProfilerMarkers.h"
#endif
#if defined(MOZ_USE_HWDECODE) && defined(MOZ_WIDGET_GTK)
# include "H264.h"
# include "mozilla/gfx/gfxVars.h"
# include "mozilla/layers/DMABUFSurfaceImage.h"
# include "mozilla/widget/DMABufLibWrapper.h"
# include "FFmpegVideoFramePool.h"
# include "va/va.h"
#endif
#if defined(MOZ_AV1) && \
(defined(FFVPX_VERSION) || LIBAVCODEC_VERSION_MAJOR >= 59)
# define FFMPEG_AV1_DECODE 1
# include "AOMDecoder.h"
#endif
#if LIBAVCODEC_VERSION_MAJOR < 54
# define AV_PIX_FMT_YUV420P PIX_FMT_YUV420P
# define AV_PIX_FMT_YUVJ420P PIX_FMT_YUVJ420P
# define AV_PIX_FMT_YUV420P10LE PIX_FMT_YUV420P10LE
# define AV_PIX_FMT_YUV422P PIX_FMT_YUV422P
# define AV_PIX_FMT_YUV422P10LE PIX_FMT_YUV422P10LE
# define AV_PIX_FMT_YUV444P PIX_FMT_YUV444P
# define AV_PIX_FMT_YUVJ444P PIX_FMT_YUVJ444P
# define AV_PIX_FMT_YUV444P10LE PIX_FMT_YUV444P10LE
# define AV_PIX_FMT_GBRP PIX_FMT_GBRP
# define AV_PIX_FMT_GBRP10LE PIX_FMT_GBRP10LE
# define AV_PIX_FMT_NONE PIX_FMT_NONE
# define AV_PIX_FMT_VAAPI_VLD PIX_FMT_VAAPI_VLD
#endif
#if LIBAVCODEC_VERSION_MAJOR > 58
# define AV_PIX_FMT_VAAPI_VLD AV_PIX_FMT_VAAPI
#endif
#include "mozilla/PodOperations.h"
#include "mozilla/StaticPrefs_gfx.h"
#include "mozilla/StaticPrefs_media.h"
#include "mozilla/TaskQueue.h"
#include "nsThreadUtils.h"
#include "prsystem.h"
#ifdef XP_WIN
# include "mozilla/gfx/DeviceManagerDx.h"
# include "mozilla/gfx/gfxVars.h"
#endif
#ifdef MOZ_ENABLE_D3D11VA
# include "D3D11TextureWrapper.h"
# include "DXVA2Manager.h"
# include "ffvpx/hwcontext_d3d11va.h"
#endif
// Forward declare from va.h
#if defined(MOZ_USE_HWDECODE) && defined(MOZ_WIDGET_GTK)
typedef int VAStatus;
# define VA_EXPORT_SURFACE_READ_ONLY 0x0001
# define VA_EXPORT_SURFACE_SEPARATE_LAYERS 0x0004
# define VA_STATUS_SUCCESS 0x00000000
#endif
// Use some extra HW frames for potential rendering lags.
// AV1 and VP9 can have maximum 8 frames for reference frames, so 1 base + 8
// references.
#define EXTRA_HW_FRAMES 9
#if LIBAVCODEC_VERSION_MAJOR >= 57 && LIBAVUTIL_VERSION_MAJOR >= 56
# define CUSTOMIZED_BUFFER_ALLOCATION 1
#endif
#define AV_LOG_DEBUG 48
typedef mozilla::layers::Image Image;
typedef mozilla::layers::PlanarYCbCrImage PlanarYCbCrImage;
namespace mozilla {
#if defined(MOZ_USE_HWDECODE) && defined(MOZ_WIDGET_GTK)
MOZ_RUNINIT nsTArray<AVCodecID>
FFmpegVideoDecoder<LIBAV_VER>::mAcceleratedFormats;
#endif
using media::TimeUnit;
/**
* FFmpeg calls back to this function with a list of pixel formats it supports.
* We choose a pixel format that we support and return it.
* For now, we just look for YUV420P, YUVJ420P, YUV444 and YUVJ444 as
* those are the only non-HW accelerated format supported by FFmpeg's H264 and
* VP9 decoder.
*/
static AVPixelFormat ChoosePixelFormat(AVCodecContext* aCodecContext,
const AVPixelFormat* aFormats) {
FFMPEGV_LOG("Choosing FFmpeg pixel format for video decoding.");
for (; *aFormats > -1; aFormats++) {
switch (*aFormats) {
case AV_PIX_FMT_YUV420P:
FFMPEGV_LOG("Requesting pixel format YUV420P.");
return AV_PIX_FMT_YUV420P;
case AV_PIX_FMT_YUVJ420P:
FFMPEGV_LOG("Requesting pixel format YUVJ420P.");
return AV_PIX_FMT_YUVJ420P;
case AV_PIX_FMT_YUV420P10LE:
FFMPEGV_LOG("Requesting pixel format YUV420P10LE.");
return AV_PIX_FMT_YUV420P10LE;
case AV_PIX_FMT_YUV422P:
FFMPEGV_LOG("Requesting pixel format YUV422P.");
return AV_PIX_FMT_YUV422P;
case AV_PIX_FMT_YUV422P10LE:
FFMPEGV_LOG("Requesting pixel format YUV422P10LE.");
return AV_PIX_FMT_YUV422P10LE;
case AV_PIX_FMT_YUV444P:
FFMPEGV_LOG("Requesting pixel format YUV444P.");
return AV_PIX_FMT_YUV444P;
case AV_PIX_FMT_YUVJ444P:
FFMPEGV_LOG("Requesting pixel format YUVJ444P.");
return AV_PIX_FMT_YUVJ444P;
case AV_PIX_FMT_YUV444P10LE:
FFMPEGV_LOG("Requesting pixel format YUV444P10LE.");
return AV_PIX_FMT_YUV444P10LE;
#if LIBAVCODEC_VERSION_MAJOR >= 57
case AV_PIX_FMT_YUV420P12LE:
FFMPEGV_LOG("Requesting pixel format YUV420P12LE.");
return AV_PIX_FMT_YUV420P12LE;
case AV_PIX_FMT_YUV422P12LE:
FFMPEGV_LOG("Requesting pixel format YUV422P12LE.");
return AV_PIX_FMT_YUV422P12LE;
case AV_PIX_FMT_YUV444P12LE:
FFMPEGV_LOG("Requesting pixel format YUV444P12LE.");
return AV_PIX_FMT_YUV444P12LE;
#endif
case AV_PIX_FMT_GBRP:
FFMPEGV_LOG("Requesting pixel format GBRP.");
return AV_PIX_FMT_GBRP;
case AV_PIX_FMT_GBRP10LE:
FFMPEGV_LOG("Requesting pixel format GBRP10LE.");
return AV_PIX_FMT_GBRP10LE;
default:
break;
}
}
NS_WARNING("FFmpeg does not share any supported pixel formats.");
return AV_PIX_FMT_NONE;
}
#ifdef MOZ_USE_HWDECODE
static AVPixelFormat ChooseVAAPIPixelFormat(AVCodecContext* aCodecContext,
const AVPixelFormat* aFormats) {
FFMPEGV_LOG("Choosing FFmpeg pixel format for VA-API video decoding.");
for (; *aFormats > -1; aFormats++) {
switch (*aFormats) {
case AV_PIX_FMT_VAAPI_VLD:
FFMPEGV_LOG("Requesting pixel format VAAPI_VLD");
return AV_PIX_FMT_VAAPI_VLD;
default:
break;
}
}
NS_WARNING("FFmpeg does not share any supported pixel formats.");
return AV_PIX_FMT_NONE;
}
static AVPixelFormat ChooseV4L2PixelFormat(AVCodecContext* aCodecContext,
const AVPixelFormat* aFormats) {
FFMPEGV_LOG("Choosing FFmpeg pixel format for V4L2 video decoding.");
for (; *aFormats > -1; aFormats++) {
switch (*aFormats) {
case AV_PIX_FMT_DRM_PRIME:
FFMPEGV_LOG("Requesting pixel format DRM PRIME");
return AV_PIX_FMT_DRM_PRIME;
default:
break;
}
}
NS_WARNING("FFmpeg does not share any supported V4L2 pixel formats.");
return AV_PIX_FMT_NONE;
}
static AVPixelFormat ChooseD3D11VAPixelFormat(AVCodecContext* aCodecContext,
const AVPixelFormat* aFormats) {
# ifdef MOZ_ENABLE_D3D11VA
FFMPEGV_LOG("Choosing FFmpeg pixel format for D3D11VA video decoding %d. ",
*aFormats);
for (; *aFormats > -1; aFormats++) {
switch (*aFormats) {
case AV_PIX_FMT_D3D11:
FFMPEGV_LOG("Requesting pixel format D3D11");
return AV_PIX_FMT_D3D11;
default:
break;
}
}
NS_WARNING("FFmpeg does not share any supported D3D11 pixel formats.");
# endif // MOZ_ENABLE_D3D11VA
return AV_PIX_FMT_NONE;
}
#endif
#if defined(MOZ_USE_HWDECODE) && defined(MOZ_WIDGET_GTK)
AVCodec* FFmpegVideoDecoder<LIBAV_VER>::FindVAAPICodec() {
AVCodec* decoder = FindHardwareAVCodec(mLib, mCodecID);
if (!decoder) {
FFMPEG_LOG(" We're missing hardware accelerated decoder");
return nullptr;
}
for (int i = 0;; i++) {
const AVCodecHWConfig* config = mLib->avcodec_get_hw_config(decoder, i);
if (!config) {
break;
}
if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX &&
config->device_type == AV_HWDEVICE_TYPE_VAAPI) {
return decoder;
}
}
FFMPEG_LOG(" HW Decoder does not support VAAPI device type");
return nullptr;
}
template <int V>
class VAAPIDisplayHolder {};
template <>
class VAAPIDisplayHolder<LIBAV_VER>;
template <>
class VAAPIDisplayHolder<LIBAV_VER> {
public:
VAAPIDisplayHolder(FFmpegLibWrapper* aLib, VADisplay aDisplay, int aDRMFd)
: mLib(aLib), mDisplay(aDisplay), mDRMFd(aDRMFd) {};
~VAAPIDisplayHolder() {
mLib->vaTerminate(mDisplay);
close(mDRMFd);
}
private:
FFmpegLibWrapper* mLib;
VADisplay mDisplay;
int mDRMFd;
};
static void VAAPIDisplayReleaseCallback(struct AVHWDeviceContext* hwctx) {
auto displayHolder =
static_cast<VAAPIDisplayHolder<LIBAV_VER>*>(hwctx->user_opaque);
delete displayHolder;
}
bool FFmpegVideoDecoder<LIBAV_VER>::CreateVAAPIDeviceContext() {
mVAAPIDeviceContext = mLib->av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_VAAPI);
if (!mVAAPIDeviceContext) {
FFMPEG_LOG(" av_hwdevice_ctx_alloc failed.");
return false;
}
auto releaseVAAPIcontext =
MakeScopeExit([&] { mLib->av_buffer_unref(&mVAAPIDeviceContext); });
AVHWDeviceContext* hwctx = (AVHWDeviceContext*)mVAAPIDeviceContext->data;
AVVAAPIDeviceContext* vactx = (AVVAAPIDeviceContext*)hwctx->hwctx;
int drmFd = widget::GetDMABufDevice()->OpenDRMFd();
mDisplay = mLib->vaGetDisplayDRM(drmFd);
if (!mDisplay) {
FFMPEG_LOG(" Can't get DRM VA-API display.");
return false;
}
hwctx->user_opaque = new VAAPIDisplayHolder<LIBAV_VER>(mLib, mDisplay, drmFd);
hwctx->free = VAAPIDisplayReleaseCallback;
int major, minor;
int status = mLib->vaInitialize(mDisplay, &major, &minor);
if (status != VA_STATUS_SUCCESS) {
FFMPEG_LOG(" vaInitialize failed.");
return false;
}
vactx->display = mDisplay;
if (mLib->av_hwdevice_ctx_init(mVAAPIDeviceContext) < 0) {
FFMPEG_LOG(" av_hwdevice_ctx_init failed.");
return false;
}
mCodecContext->hw_device_ctx = mLib->av_buffer_ref(mVAAPIDeviceContext);
releaseVAAPIcontext.release();
return true;
}
void FFmpegVideoDecoder<LIBAV_VER>::AdjustHWDecodeLogging() {
if (!getenv("MOZ_AV_LOG_LEVEL") &&
MOZ_LOG_TEST(sFFmpegVideoLog, LogLevel::Debug)) {
mLib->av_log_set_level(AV_LOG_DEBUG);
}
if (!getenv("LIBVA_MESSAGING_LEVEL")) {
if (MOZ_LOG_TEST(sFFmpegVideoLog, LogLevel::Debug)) {
setenv("LIBVA_MESSAGING_LEVEL", "1", false);
} else if (MOZ_LOG_TEST(sFFmpegVideoLog, LogLevel::Info)) {
setenv("LIBVA_MESSAGING_LEVEL", "2", false);
} else {
setenv("LIBVA_MESSAGING_LEVEL", "0", false);
}
}
}
MediaResult FFmpegVideoDecoder<LIBAV_VER>::InitVAAPIDecoder() {
FFMPEG_LOG("Initialising VA-API FFmpeg decoder");
StaticMutexAutoLock mon(sMutex);
// mAcceleratedFormats is already configured so check supported
// formats before we do anything.
if (mAcceleratedFormats.Length()) {
if (!IsFormatAccelerated(mCodecID)) {
FFMPEG_LOG(" Format %s is not accelerated",
mLib->avcodec_get_name(mCodecID));
return NS_ERROR_NOT_AVAILABLE;
} else {
FFMPEG_LOG(" Format %s is accelerated",
mLib->avcodec_get_name(mCodecID));
}
}
if (!mLib->IsVAAPIAvailable()) {
FFMPEG_LOG(" libva library or symbols are missing.");
return NS_ERROR_NOT_AVAILABLE;
}
AVCodec* codec = FindVAAPICodec();
if (!codec) {
FFMPEG_LOG(" couldn't find ffmpeg VA-API decoder");
return NS_ERROR_DOM_MEDIA_FATAL_ERR;
}
// This logic is mirrored in FFmpegDecoderModule::Supports. We prefer to use
// our own OpenH264 decoder through the plugin over ffmpeg by default due to
// broken decoding with some versions. openh264 has broken decoding of some
// h264 videos so don't use it unless explicitly allowed for now.
if (!strcmp(codec->name, "libopenh264") &&
!StaticPrefs::media_ffmpeg_allow_openh264()) {
FFMPEG_LOG(" unable to find codec (openh264 disabled by pref)");
return MediaResult(
NS_ERROR_DOM_MEDIA_FATAL_ERR,
RESULT_DETAIL("unable to find codec (openh264 disabled by pref)"));
}
FFMPEG_LOG(" codec %s : %s", codec->name, codec->long_name);
if (!(mCodecContext = mLib->avcodec_alloc_context3(codec))) {
FFMPEG_LOG(" couldn't init VA-API ffmpeg context");
return NS_ERROR_OUT_OF_MEMORY;
}
mCodecContext->opaque = this;
InitHWCodecContext(ContextType::VAAPI);
auto releaseVAAPIdecoder = MakeScopeExit([&] {
if (mVAAPIDeviceContext) {
mLib->av_buffer_unref(&mVAAPIDeviceContext);
}
if (mCodecContext) {
mLib->av_freep(&mCodecContext);
}
});
if (!CreateVAAPIDeviceContext()) {
mLib->av_freep(&mCodecContext);
FFMPEG_LOG(" Failed to create VA-API device context");
return NS_ERROR_DOM_MEDIA_FATAL_ERR;
}
MediaResult ret = AllocateExtraData();
if (NS_FAILED(ret)) {
mLib->av_buffer_unref(&mVAAPIDeviceContext);
mLib->av_freep(&mCodecContext);
return ret;
}
if (mLib->avcodec_open2(mCodecContext, codec, nullptr) < 0) {
mLib->av_buffer_unref(&mVAAPIDeviceContext);
mLib->av_freep(&mCodecContext);
FFMPEG_LOG(" Couldn't initialise VA-API decoder");
return NS_ERROR_DOM_MEDIA_FATAL_ERR;
}
if (mAcceleratedFormats.IsEmpty()) {
mAcceleratedFormats = GetAcceleratedFormats();
if (!IsFormatAccelerated(mCodecID)) {
FFMPEG_LOG(" Format %s is not accelerated",
mLib->avcodec_get_name(mCodecID));
return NS_ERROR_NOT_AVAILABLE;
}
}
AdjustHWDecodeLogging();
FFMPEG_LOG(" VA-API FFmpeg init successful");
releaseVAAPIdecoder.release();
return NS_OK;
}
MediaResult FFmpegVideoDecoder<LIBAV_VER>::InitV4L2Decoder() {
FFMPEG_LOG("Initialising V4L2-DRM FFmpeg decoder");
StaticMutexAutoLock mon(sMutex);
// mAcceleratedFormats is already configured so check supported
// formats before we do anything.
if (mAcceleratedFormats.Length()) {
if (!IsFormatAccelerated(mCodecID)) {
FFMPEG_LOG(" Format %s is not accelerated",
mLib->avcodec_get_name(mCodecID));
return NS_ERROR_NOT_AVAILABLE;
}
FFMPEG_LOG(" Format %s is accelerated", mLib->avcodec_get_name(mCodecID));
}
// Select the appropriate v4l2 codec
AVCodec* codec = nullptr;
if (mCodecID == AV_CODEC_ID_H264) {
codec = mLib->avcodec_find_decoder_by_name("h264_v4l2m2m");
}
if (!codec) {
FFMPEG_LOG("No appropriate v4l2 codec found");
return NS_ERROR_DOM_MEDIA_FATAL_ERR;
}
FFMPEG_LOG(" V4L2 codec %s : %s", codec->name, codec->long_name);
if (!(mCodecContext = mLib->avcodec_alloc_context3(codec))) {
FFMPEG_LOG(" couldn't init HW ffmpeg context");
return NS_ERROR_OUT_OF_MEMORY;
}
mCodecContext->opaque = this;
InitHWCodecContext(ContextType::V4L2);
// Disable cropping in FFmpeg. Because our frames are opaque DRM buffers
// FFmpeg can't actually crop them and it tries to do so by just modifying
// the width and height. This causes problems because V4L2 outputs a single
// buffer/layer/plane with all three planes stored contiguously. We need to
// know the offsets to each plane, and if FFmpeg applies cropping (and then
// we can't find out what the original uncropped width/height was) then we
// can't work out the offsets.
mCodecContext->apply_cropping = 0;
auto releaseDecoder = MakeScopeExit([&] {
if (mCodecContext) {
mLib->av_freep(&mCodecContext);
}
});
MediaResult ret = AllocateExtraData();
if (NS_FAILED(ret)) {
mLib->av_freep(&mCodecContext);
return ret;
}
if (mLib->avcodec_open2(mCodecContext, codec, nullptr) < 0) {
mLib->av_freep(&mCodecContext);
FFMPEG_LOG(" Couldn't initialise V4L2 decoder");
return NS_ERROR_DOM_MEDIA_FATAL_ERR;
}
// Set mAcceleratedFormats
if (mAcceleratedFormats.IsEmpty()) {
// FFmpeg does not correctly report that the V4L2 wrapper decoders are
// hardware accelerated, but we know they always are. If we've gotten
// this far then we know this codec has a V4L2 wrapper decoder and so is
// accelerateed.
mAcceleratedFormats.AppendElement(mCodecID);
}
AdjustHWDecodeLogging();
FFMPEG_LOG(" V4L2 FFmpeg init successful");
mUsingV4L2 = true;
releaseDecoder.release();
return NS_OK;
}
#endif
#if LIBAVCODEC_VERSION_MAJOR < 58
FFmpegVideoDecoder<LIBAV_VER>::PtsCorrectionContext::PtsCorrectionContext()
: mNumFaultyPts(0),
mNumFaultyDts(0),
mLastPts(INT64_MIN),
mLastDts(INT64_MIN) {}
int64_t FFmpegVideoDecoder<LIBAV_VER>::PtsCorrectionContext::GuessCorrectPts(
int64_t aPts, int64_t aDts) {
int64_t pts = AV_NOPTS_VALUE;
if (aDts != int64_t(AV_NOPTS_VALUE)) {
mNumFaultyDts += aDts <= mLastDts;
mLastDts = aDts;
}
if (aPts != int64_t(AV_NOPTS_VALUE)) {
mNumFaultyPts += aPts <= mLastPts;
mLastPts = aPts;
}
if ((mNumFaultyPts <= mNumFaultyDts || aDts == int64_t(AV_NOPTS_VALUE)) &&
aPts != int64_t(AV_NOPTS_VALUE)) {
pts = aPts;
} else {
pts = aDts;
}
return pts;
}
void FFmpegVideoDecoder<LIBAV_VER>::PtsCorrectionContext::Reset() {
mNumFaultyPts = 0;
mNumFaultyDts = 0;
mLastPts = INT64_MIN;
mLastDts = INT64_MIN;
}
#endif
#if defined(MOZ_USE_HWDECODE) && defined(MOZ_WIDGET_GTK)
bool FFmpegVideoDecoder<LIBAV_VER>::ShouldEnableLinuxHWDecoding() const {
bool supported = false;
switch (mCodecID) {
case AV_CODEC_ID_H264:
supported = gfx::gfxVars::UseH264HwDecode();
break;
case AV_CODEC_ID_VP8:
supported = gfx::gfxVars::UseVP8HwDecode();
break;
case AV_CODEC_ID_VP9:
supported = gfx::gfxVars::UseVP9HwDecode();
break;
case AV_CODEC_ID_AV1:
supported = gfx::gfxVars::UseAV1HwDecode();
break;
default:
break;
}
if (!supported) {
FFMPEG_LOG("Codec %s is not accelerated", mLib->avcodec_get_name(mCodecID));
return false;
}
bool isHardwareWebRenderUsed = mImageAllocator &&
(mImageAllocator->GetCompositorBackendType() ==
layers::LayersBackend::LAYERS_WR) &&
!mImageAllocator->UsingSoftwareWebRender();
if (!isHardwareWebRenderUsed) {
FFMPEG_LOG("Hardware WebRender is off, VAAPI is disabled");
return false;
}
if (!XRE_IsRDDProcess()) {
FFMPEG_LOG("VA-API works in RDD process only");
return false;
}
return true;
}
#endif
FFmpegVideoDecoder<LIBAV_VER>::FFmpegVideoDecoder(
FFmpegLibWrapper* aLib, const VideoInfo& aConfig,
KnowsCompositor* aAllocator, ImageContainer* aImageContainer,
bool aLowLatency, bool aDisableHardwareDecoding,
Maybe<TrackingId> aTrackingId)
: FFmpegDataDecoder(aLib, GetCodecId(aConfig.mMimeType)),
mImageAllocator(aAllocator),
#ifdef MOZ_USE_HWDECODE
# ifdef MOZ_WIDGET_GTK
mHardwareDecodingDisabled(aDisableHardwareDecoding ||
!ShouldEnableLinuxHWDecoding()),
# else
mHardwareDecodingDisabled(aDisableHardwareDecoding),
# endif // MOZ_WIDGET_GTK
#endif // MOZ_USE_HWDECODE
mImageContainer(aImageContainer),
mInfo(aConfig),
mLowLatency(aLowLatency),
mTrackingId(std::move(aTrackingId)) {
FFMPEG_LOG("FFmpegVideoDecoder::FFmpegVideoDecoder MIME %s Codec ID %d",
aConfig.mMimeType.get(), mCodecID);
// Use a new MediaByteBuffer as the object will be modified during
// initialization.
mExtraData = new MediaByteBuffer;
mExtraData->AppendElements(*aConfig.mExtraData);
#ifdef MOZ_USE_HWDECODE
InitHWDecoderIfAllowed();
#endif // MOZ_USE_HWDECODE
}
FFmpegVideoDecoder<LIBAV_VER>::~FFmpegVideoDecoder() {
#ifdef CUSTOMIZED_BUFFER_ALLOCATION
MOZ_DIAGNOSTIC_ASSERT(mAllocatedImages.IsEmpty(),
"Should release all shmem buffers before destroy!");
#endif
}
#ifdef MOZ_USE_HWDECODE
void FFmpegVideoDecoder<LIBAV_VER>::InitHWDecoderIfAllowed() {
if (mHardwareDecodingDisabled) {
return;
}
# ifdef MOZ_ENABLE_VAAPI
if (NS_SUCCEEDED(InitVAAPIDecoder())) {
return;
}
# endif // MOZ_ENABLE_VAAPI
# ifdef MOZ_ENABLE_V4L2
// VAAPI didn't work or is disabled, so try V4L2 with DRM
if (NS_SUCCEEDED(InitV4L2Decoder())) {
return;
}
# endif // MOZ_ENABLE_V4L2
# ifdef MOZ_ENABLE_D3D11VA
if (XRE_IsGPUProcess() && NS_SUCCEEDED(InitD3D11VADecoder())) {
return;
}
# endif // MOZ_ENABLE_D3D11VA
}
#endif // MOZ_USE_HWDECODE
RefPtr<MediaDataDecoder::InitPromise> FFmpegVideoDecoder<LIBAV_VER>::Init() {
FFMPEG_LOG("FFmpegVideoDecoder, init, IsHardwareAccelerated=%d\n",
IsHardwareAccelerated());
// We've finished the HW decoder initialization in the ctor.
if (IsHardwareAccelerated()) {
return InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, __func__);
}
MediaResult rv = InitSWDecoder(nullptr);
return NS_SUCCEEDED(rv)
? InitPromise::CreateAndResolve(TrackInfo::kVideoTrack, __func__)
: InitPromise::CreateAndReject(rv, __func__);
}
static gfx::ColorRange GetColorRange(enum AVColorRange& aColorRange) {
return aColorRange == AVCOL_RANGE_JPEG ? gfx::ColorRange::FULL
: gfx::ColorRange::LIMITED;
}
static bool IsYUVFormat(const AVPixelFormat& aFormat) {
return aFormat != AV_PIX_FMT_GBRP && aFormat != AV_PIX_FMT_GBRP10LE;
}
static gfx::YUVColorSpace TransferAVColorSpaceToColorSpace(
const AVColorSpace aSpace, const AVPixelFormat aFormat,
const gfx::IntSize& aSize) {
if (!IsYUVFormat(aFormat)) {
return gfx::YUVColorSpace::Identity;
}
switch (aSpace) {
#if LIBAVCODEC_VERSION_MAJOR >= 55
case AVCOL_SPC_BT2020_NCL:
case AVCOL_SPC_BT2020_CL:
return gfx::YUVColorSpace::BT2020;
#endif
case AVCOL_SPC_BT709:
return gfx::YUVColorSpace::BT709;
case AVCOL_SPC_SMPTE170M:
case AVCOL_SPC_BT470BG:
return gfx::YUVColorSpace::BT601;
default:
return DefaultColorSpace(aSize);
}
}
#ifdef CUSTOMIZED_BUFFER_ALLOCATION
static int GetVideoBufferWrapper(struct AVCodecContext* aCodecContext,
AVFrame* aFrame, int aFlags) {
auto* decoder =
static_cast<FFmpegVideoDecoder<LIBAV_VER>*>(aCodecContext->opaque);
int rv = decoder->GetVideoBuffer(aCodecContext, aFrame, aFlags);
return rv < 0 ? decoder->GetVideoBufferDefault(aCodecContext, aFrame, aFlags)
: rv;
}
static void ReleaseVideoBufferWrapper(void* opaque, uint8_t* data) {
if (opaque) {
FFMPEGV_LOG("ReleaseVideoBufferWrapper: PlanarYCbCrImage=%p", opaque);
RefPtr<ImageBufferWrapper> image = static_cast<ImageBufferWrapper*>(opaque);
image->ReleaseBuffer();
}
}
static bool IsColorFormatSupportedForUsingCustomizedBuffer(
const AVPixelFormat& aFormat) {
# if XP_WIN
// Currently the web render doesn't support uploading R16 surface, so we can't
// use the shmem texture for 10 bit+ videos which would be uploaded by the
// web render. See Bug 1751498.
return aFormat == AV_PIX_FMT_YUV420P || aFormat == AV_PIX_FMT_YUVJ420P ||
aFormat == AV_PIX_FMT_YUV444P || aFormat == AV_PIX_FMT_YUVJ444P;
# else
// For now, we only support for YUV420P, YUVJ420P, YUV444P and YUVJ444P which
// are the only non-HW accelerated format supported by FFmpeg's H264 and VP9
// decoder.
return aFormat == AV_PIX_FMT_YUV420P || aFormat == AV_PIX_FMT_YUVJ420P ||
aFormat == AV_PIX_FMT_YUV420P10LE ||
aFormat == AV_PIX_FMT_YUV420P12LE || aFormat == AV_PIX_FMT_YUV444P ||
aFormat == AV_PIX_FMT_YUVJ444P || aFormat == AV_PIX_FMT_YUV444P10LE ||
aFormat == AV_PIX_FMT_YUV444P12LE;
# endif
}
static bool IsYUV420Sampling(const AVPixelFormat& aFormat) {
return aFormat == AV_PIX_FMT_YUV420P || aFormat == AV_PIX_FMT_YUVJ420P ||
aFormat == AV_PIX_FMT_YUV420P10LE || aFormat == AV_PIX_FMT_YUV420P12LE;
}
layers::TextureClient*
FFmpegVideoDecoder<LIBAV_VER>::AllocateTextureClientForImage(
struct AVCodecContext* aCodecContext, PlanarYCbCrImage* aImage) {
MOZ_ASSERT(
IsColorFormatSupportedForUsingCustomizedBuffer(aCodecContext->pix_fmt));
// FFmpeg will store images with color depth > 8 bits in 16 bits with extra
// padding.
const int32_t bytesPerChannel =
GetColorDepth(aCodecContext->pix_fmt) == gfx::ColorDepth::COLOR_8 ? 1 : 2;
// If adjusted Ysize is larger than the actual image size (coded_width *
// coded_height), that means ffmpeg decoder needs extra padding on both width
// and height. If that happens, the planes will need to be cropped later in
// order to avoid visible incorrect border on the right and bottom of the
// actual image.
//
// Here are examples of various sizes video in YUV420P format, the width and
// height would need to be adjusted in order to align padding.
//
// Eg1. video (1920*1080)
// plane Y
// width 1920 height 1080 -> adjusted-width 1920 adjusted-height 1088
// plane Cb/Cr
// width 960 height 540 -> adjusted-width 1024 adjusted-height 544
//
// Eg2. video (2560*1440)
// plane Y
// width 2560 height 1440 -> adjusted-width 2560 adjusted-height 1440
// plane Cb/Cr
// width 1280 height 720 -> adjusted-width 1280 adjusted-height 736
layers::PlanarYCbCrData data;
const auto yDims =
gfx::IntSize{aCodecContext->coded_width, aCodecContext->coded_height};
auto paddedYSize = yDims;
mLib->avcodec_align_dimensions(aCodecContext, &paddedYSize.width,
&paddedYSize.height);
data.mYStride = paddedYSize.Width() * bytesPerChannel;
MOZ_ASSERT(
IsColorFormatSupportedForUsingCustomizedBuffer(aCodecContext->pix_fmt));
auto uvDims = yDims;
if (IsYUV420Sampling(aCodecContext->pix_fmt)) {
uvDims.width = (uvDims.width + 1) / 2;
uvDims.height = (uvDims.height + 1) / 2;
data.mChromaSubsampling = gfx::ChromaSubsampling::HALF_WIDTH_AND_HEIGHT;
}
auto paddedCbCrSize = uvDims;
mLib->avcodec_align_dimensions(aCodecContext, &paddedCbCrSize.width,
&paddedCbCrSize.height);
data.mCbCrStride = paddedCbCrSize.Width() * bytesPerChannel;
// Setting other attributes
data.mPictureRect = gfx::IntRect(
mInfo.ScaledImageRect(aCodecContext->width, aCodecContext->height)
.TopLeft(),
gfx::IntSize(aCodecContext->width, aCodecContext->height));
data.mStereoMode = mInfo.mStereoMode;
if (aCodecContext->colorspace != AVCOL_SPC_UNSPECIFIED) {
data.mYUVColorSpace = TransferAVColorSpaceToColorSpace(
aCodecContext->colorspace, aCodecContext->pix_fmt,
data.mPictureRect.Size());
} else {
data.mYUVColorSpace = mInfo.mColorSpace
? *mInfo.mColorSpace
: DefaultColorSpace(data.mPictureRect.Size());
}
data.mColorDepth = GetColorDepth(aCodecContext->pix_fmt);
data.mColorRange = GetColorRange(aCodecContext->color_range);
FFMPEG_LOGV(
"Created plane data, YSize=(%d, %d), CbCrSize=(%d, %d), "
"CroppedYSize=(%d, %d), CroppedCbCrSize=(%d, %d), ColorDepth=%hhu",
paddedYSize.Width(), paddedYSize.Height(), paddedCbCrSize.Width(),
paddedCbCrSize.Height(), data.YPictureSize().Width(),
data.YPictureSize().Height(), data.CbCrPictureSize().Width(),
data.CbCrPictureSize().Height(), static_cast<uint8_t>(data.mColorDepth));
// Allocate a shmem buffer for image.
if (NS_FAILED(aImage->CreateEmptyBuffer(data, paddedYSize, paddedCbCrSize))) {
return nullptr;
}
return aImage->GetTextureClient(mImageAllocator);
}
int FFmpegVideoDecoder<LIBAV_VER>::GetVideoBuffer(
struct AVCodecContext* aCodecContext, AVFrame* aFrame, int aFlags) {
FFMPEG_LOGV("GetVideoBuffer: aCodecContext=%p aFrame=%p", aCodecContext,
aFrame);
if (!StaticPrefs::media_ffmpeg_customized_buffer_allocation()) {
return AVERROR(EINVAL);
}
if (mIsUsingShmemBufferForDecode && !*mIsUsingShmemBufferForDecode) {
return AVERROR(EINVAL);
}
// Codec doesn't support custom allocator.
if (!(aCodecContext->codec->capabilities & AV_CODEC_CAP_DR1)) {
return AVERROR(EINVAL);
}
// Pre-allocation is only for sw decoding. During decoding, ffmpeg decoder
// will need to reference decoded frames, if those frames are on shmem buffer,
// then it would cause a need to read CPU data from GPU, which is slow.
if (IsHardwareAccelerated()) {
return AVERROR(EINVAL);
}
if (!IsColorFormatSupportedForUsingCustomizedBuffer(aCodecContext->pix_fmt)) {
FFMPEG_LOG("Not support color format %d", aCodecContext->pix_fmt);
return AVERROR(EINVAL);
}
if (aCodecContext->lowres != 0) {
FFMPEG_LOG("Not support low resolution decoding");
return AVERROR(EINVAL);
}
const gfx::IntSize size(aCodecContext->width, aCodecContext->height);
int rv = mLib->av_image_check_size(size.Width(), size.Height(), 0, nullptr);
if (rv < 0) {
FFMPEG_LOG("Invalid image size");
return rv;
}
CheckedInt32 dataSize = mLib->av_image_get_buffer_size(
aCodecContext->pix_fmt, aCodecContext->coded_width,
aCodecContext->coded_height, 32);
if (!dataSize.isValid()) {
FFMPEG_LOG("Data size overflow!");
return AVERROR(EINVAL);
}
if (!mImageContainer) {
FFMPEG_LOG("No Image container!");
return AVERROR(EINVAL);
}
RefPtr<PlanarYCbCrImage> image = mImageContainer->CreatePlanarYCbCrImage();
if (!image) {