Source code
Revision control
Copy as Markdown
Other Tools
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set ts=2 sw=2 sts=2 et cindent: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
#include "MediaChangeMonitor.h"
#include "AnnexB.h"
#include "H264.h"
#include "H265.h"
#include "GeckoProfiler.h"
#include "ImageContainer.h"
#include "MP4Decoder.h"
#include "MediaInfo.h"
#include "PDMFactory.h"
#include "VPXDecoder.h"
#ifdef MOZ_AV1
# include "AOMDecoder.h"
#endif
#include "gfxUtils.h"
#include "mozilla/ProfilerMarkers.h"
#include "mozilla/StaticPrefs_media.h"
#include "mozilla/TaskQueue.h"
namespace mozilla {
extern LazyLogModule gMediaDecoderLog;
#define LOG(x, ...) \
MOZ_LOG(gMediaDecoderLog, LogLevel::Debug, (x, ##__VA_ARGS__))
// H264ChangeMonitor is used to ensure that only AVCC or AnnexB is fed to the
// underlying MediaDataDecoder. The H264ChangeMonitor allows playback of content
// where the SPS NAL may not be provided in the init segment (e.g. AVC3 or Annex
// B) H264ChangeMonitor will monitor the input data, and will delay creation of
// the MediaDataDecoder until a SPS and PPS NALs have been extracted.
class H264ChangeMonitor : public MediaChangeMonitor::CodecChangeMonitor {
public:
explicit H264ChangeMonitor(const VideoInfo& aInfo, bool aFullParsing)
: mCurrentConfig(aInfo), mFullParsing(aFullParsing) {
if (CanBeInstantiated()) {
UpdateConfigFromExtraData(aInfo.mExtraData);
}
}
bool CanBeInstantiated() const override {
return H264::HasSPS(mCurrentConfig.mExtraData);
}
MediaResult CheckForChange(MediaRawData* aSample) override {
// To be usable we need to convert the sample to 4 bytes NAL size AVCC.
if (!AnnexB::ConvertSampleToAVCC(aSample)) {
// We need AVCC content to be able to later parse the SPS.
// This is a no-op if the data is already AVCC.
return MediaResult(NS_ERROR_OUT_OF_MEMORY,
RESULT_DETAIL("ConvertSampleToAVCC"));
}
if (!AnnexB::IsAVCC(aSample)) {
return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
RESULT_DETAIL("Invalid H264 content"));
}
RefPtr<MediaByteBuffer> extra_data =
aSample->mKeyframe || !mGotSPS || mFullParsing
? H264::ExtractExtraData(aSample)
: nullptr;
if (!H264::HasSPS(extra_data) && !H264::HasSPS(mCurrentConfig.mExtraData)) {
// We don't have inband data and the original config didn't contain a SPS.
// We can't decode this content.
return NS_ERROR_NOT_INITIALIZED;
}
mGotSPS = true;
if (!H264::HasSPS(extra_data)) {
// This sample doesn't contain inband SPS/PPS
// We now check if the out of band one has changed.
// This scenario can currently only occur on Android with devices that can
// recycle a decoder.
bool hasOutOfBandExtraData = H264::HasSPS(aSample->mExtraData);
if (!hasOutOfBandExtraData || !mPreviousExtraData ||
H264::CompareExtraData(aSample->mExtraData, mPreviousExtraData)) {
if (hasOutOfBandExtraData && !mPreviousExtraData) {
// We are decoding the first sample, store the out of band sample's
// extradata so that we can check for future change.
mPreviousExtraData = aSample->mExtraData;
}
return NS_OK;
}
extra_data = aSample->mExtraData;
} else if (H264::CompareExtraData(extra_data, mCurrentConfig.mExtraData)) {
return NS_OK;
}
// Store the sample's extradata so we don't trigger a false positive
// with the out of band test on the next sample.
mPreviousExtraData = aSample->mExtraData;
UpdateConfigFromExtraData(extra_data);
PROFILER_MARKER_TEXT("H264 Stream Change", MEDIA_PLAYBACK, {},
"H264ChangeMonitor::CheckForChange has detected a "
"change in the stream and will request a new decoder");
return NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER;
}
const TrackInfo& Config() const override { return mCurrentConfig; }
MediaResult PrepareSample(MediaDataDecoder::ConversionRequired aConversion,
MediaRawData* aSample,
bool aNeedKeyFrame) override {
MOZ_DIAGNOSTIC_ASSERT(
aConversion == MediaDataDecoder::ConversionRequired::kNeedAnnexB ||
aConversion == MediaDataDecoder::ConversionRequired::kNeedAVCC,
"Conversion must be either AVCC or AnnexB");
aSample->mExtraData = mCurrentConfig.mExtraData;
aSample->mTrackInfo = mTrackInfo;
if (aConversion == MediaDataDecoder::ConversionRequired::kNeedAnnexB) {
auto res = AnnexB::ConvertAVCCSampleToAnnexB(aSample, aNeedKeyFrame);
if (res.isErr()) {
return MediaResult(res.unwrapErr(),
RESULT_DETAIL("ConvertSampleToAnnexB"));
}
}
return NS_OK;
}
private:
void UpdateConfigFromExtraData(MediaByteBuffer* aExtraData) {
SPSData spsdata;
if (H264::DecodeSPSFromExtraData(aExtraData, spsdata) &&
spsdata.pic_width > 0 && spsdata.pic_height > 0) {
H264::EnsureSPSIsSane(spsdata);
mCurrentConfig.mImage.width = spsdata.pic_width;
mCurrentConfig.mImage.height = spsdata.pic_height;
mCurrentConfig.mDisplay.width = spsdata.display_width;
mCurrentConfig.mDisplay.height = spsdata.display_height;
mCurrentConfig.mColorDepth = spsdata.ColorDepth();
mCurrentConfig.mColorSpace = Some(spsdata.ColorSpace());
// spsdata.colour_primaries has the same values as
// gfx::CICP::ColourPrimaries.
mCurrentConfig.mColorPrimaries = gfxUtils::CicpToColorPrimaries(
static_cast<gfx::CICP::ColourPrimaries>(spsdata.colour_primaries),
gMediaDecoderLog);
// spsdata.transfer_characteristics has the same values as
// gfx::CICP::TransferCharacteristics.
mCurrentConfig.mTransferFunction = gfxUtils::CicpToTransferFunction(
static_cast<gfx::CICP::TransferCharacteristics>(
spsdata.transfer_characteristics));
mCurrentConfig.mColorRange = spsdata.video_full_range_flag
? gfx::ColorRange::FULL
: gfx::ColorRange::LIMITED;
}
mCurrentConfig.mExtraData = aExtraData;
mTrackInfo = new TrackInfoSharedPtr(mCurrentConfig, mStreamID++);
}
VideoInfo mCurrentConfig;
uint32_t mStreamID = 0;
const bool mFullParsing;
bool mGotSPS = false;
RefPtr<TrackInfoSharedPtr> mTrackInfo;
RefPtr<MediaByteBuffer> mPreviousExtraData;
};
class HEVCChangeMonitor : public MediaChangeMonitor::CodecChangeMonitor {
public:
explicit HEVCChangeMonitor(const VideoInfo& aInfo) : mCurrentConfig(aInfo) {
const bool canBeInstantiated = CanBeInstantiated();
if (canBeInstantiated) {
UpdateConfigFromExtraData(aInfo.mExtraData);
}
LOG("created HEVCChangeMonitor, CanBeInstantiated=%d", canBeInstantiated);
}
bool CanBeInstantiated() const override {
auto rv = HVCCConfig::Parse(mCurrentConfig.mExtraData);
if (rv.isErr()) {
return false;
}
return rv.unwrap().HasSPS();
}
MediaResult CheckForChange(MediaRawData* aSample) override {
// To be usable we need to convert the sample to 4 bytes NAL size HVCC.
if (auto rv = AnnexB::ConvertSampleToHVCC(aSample); rv.isErr()) {
// We need HVCC content to be able to later parse the SPS.
// This is a no-op if the data is already HVCC.
nsPrintfCString msg("Failed to convert to HVCC");
LOG("%s", msg.get());
return MediaResult(rv.unwrapErr(), msg);
}
if (!AnnexB::IsHVCC(aSample)) {
nsPrintfCString msg("Invalid HVCC content");
LOG("%s", msg.get());
return MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR, msg);
}
RefPtr<MediaByteBuffer> extraData =
aSample->mKeyframe || !mGotSPS ? H265::ExtractHVCCExtraData(aSample)
: nullptr;
// Sample doesn't contain any SPS and we already have SPS, do nothing.
auto curConfig = HVCCConfig::Parse(mCurrentConfig.mExtraData);
if ((!extraData || extraData->IsEmpty()) && curConfig.unwrap().HasSPS()) {
return NS_OK;
}
auto newConfig = HVCCConfig::Parse(extraData);
// Ignore a corrupted extradata.
if (newConfig.isErr()) {
LOG("Ignore corrupted extradata");
return NS_OK;
}
if (!newConfig.unwrap().HasSPS() && !curConfig.unwrap().HasSPS()) {
// We don't have inband data and the original config didn't contain a SPS.
// We can't decode this content.
LOG("No sps found, waiting for initialization");
return NS_ERROR_NOT_INITIALIZED;
}
mGotSPS = true;
if (H265::CompareExtraData(extraData, mCurrentConfig.mExtraData)) {
return NS_OK;
}
UpdateConfigFromExtraData(extraData);
nsPrintfCString msg(
"HEVCChangeMonitor::CheckForChange has detected a change in the stream "
"and will request a new decoder");
LOG("%s", msg.get());
PROFILER_MARKER_TEXT("HEVC Stream Change", MEDIA_PLAYBACK, {}, msg);
return NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER;
}
const TrackInfo& Config() const override { return mCurrentConfig; }
MediaResult PrepareSample(MediaDataDecoder::ConversionRequired aConversion,
MediaRawData* aSample,
bool aNeedKeyFrame) override {
MOZ_DIAGNOSTIC_ASSERT(aConversion ==
MediaDataDecoder::ConversionRequired::kNeedAnnexB);
aSample->mExtraData = mCurrentConfig.mExtraData;
aSample->mTrackInfo = mTrackInfo;
if (AnnexB::IsHVCC(aSample)) {
auto res = AnnexB::ConvertHVCCSampleToAnnexB(aSample, aNeedKeyFrame);
if (res.isErr()) {
return MediaResult(res.unwrapErr(),
RESULT_DETAIL("ConvertSampleToAnnexB"));
}
}
return NS_OK;
}
bool IsHardwareAccelerated(nsACString& aFailureReason) const override {
// We only support HEVC via hardware decoding.
return true;
}
private:
void UpdateConfigFromExtraData(MediaByteBuffer* aExtraData) {
if (auto rv = H265::DecodeSPSFromHVCCExtraData(aExtraData); rv.isOk()) {
const auto sps = rv.unwrap();
mCurrentConfig.mImage.width = sps.GetImageSize().Width();
mCurrentConfig.mImage.height = sps.GetImageSize().Height();
mCurrentConfig.mDisplay.width = sps.GetDisplaySize().Width();
mCurrentConfig.mDisplay.height = sps.GetDisplaySize().Height();
mCurrentConfig.mColorDepth = sps.ColorDepth();
mCurrentConfig.mColorSpace = Some(sps.ColorSpace());
mCurrentConfig.mColorPrimaries = gfxUtils::CicpToColorPrimaries(
static_cast<gfx::CICP::ColourPrimaries>(sps.ColorPrimaries()),
gMediaDecoderLog);
mCurrentConfig.mTransferFunction = gfxUtils::CicpToTransferFunction(
static_cast<gfx::CICP::TransferCharacteristics>(
sps.TransferFunction()));
mCurrentConfig.mColorRange = sps.IsFullColorRange()
? gfx::ColorRange::FULL
: gfx::ColorRange::LIMITED;
}
MOZ_ASSERT(HVCCConfig::Parse(aExtraData).isOk());
mCurrentConfig.mExtraData = aExtraData;
mTrackInfo = new TrackInfoSharedPtr(mCurrentConfig, mStreamID++);
}
VideoInfo mCurrentConfig;
uint32_t mStreamID = 0;
bool mGotSPS = false;
RefPtr<TrackInfoSharedPtr> mTrackInfo;
};
// Gets the pixel aspect ratio from the decoded video size and the rendered
// size.
inline double GetPixelAspectRatio(const gfx::IntSize& aImage,
const gfx::IntSize& aDisplay) {
return (static_cast<double>(aDisplay.Width()) / aImage.Width()) /
(static_cast<double>(aDisplay.Height()) / aImage.Height());
}
// Returns the render size based on the PAR and the new image size.
inline gfx::IntSize ApplyPixelAspectRatio(double aPixelAspectRatio,
const gfx::IntSize& aImage) {
return gfx::IntSize(static_cast<int32_t>(aImage.Width() * aPixelAspectRatio),
aImage.Height());
}
class VPXChangeMonitor : public MediaChangeMonitor::CodecChangeMonitor {
public:
explicit VPXChangeMonitor(const VideoInfo& aInfo)
: mCurrentConfig(aInfo),
mCodec(VPXDecoder::IsVP8(aInfo.mMimeType) ? VPXDecoder::Codec::VP8
: VPXDecoder::Codec::VP9),
mPixelAspectRatio(GetPixelAspectRatio(aInfo.mImage, aInfo.mDisplay)) {
mTrackInfo = new TrackInfoSharedPtr(mCurrentConfig, mStreamID++);
if (mCurrentConfig.mExtraData && !mCurrentConfig.mExtraData->IsEmpty()) {
// If we're passed VP codec configuration, store it so that we can
// instantiate the decoder on init.
VPXDecoder::VPXStreamInfo vpxInfo;
vpxInfo.mImage = mCurrentConfig.mImage;
vpxInfo.mDisplay = mCurrentConfig.mDisplay;
VPXDecoder::ReadVPCCBox(vpxInfo, mCurrentConfig.mExtraData);
mInfo = Some(vpxInfo);
}
}
bool CanBeInstantiated() const override {
// We want to see at least one sample before we create a decoder so that we
// can create the vpcC content on mCurrentConfig.mExtraData.
return mCodec == VPXDecoder::Codec::VP8 || mInfo ||
mCurrentConfig.mCrypto.IsEncrypted();
}
MediaResult CheckForChange(MediaRawData* aSample) override {
// Don't look at encrypted content.
if (aSample->mCrypto.IsEncrypted()) {
return NS_OK;
}
auto dataSpan = Span<const uint8_t>(aSample->Data(), aSample->Size());
// We don't trust the keyframe flag as set on the MediaRawData.
VPXDecoder::VPXStreamInfo info;
if (!VPXDecoder::GetStreamInfo(dataSpan, info, mCodec)) {
return NS_ERROR_DOM_MEDIA_DECODE_ERR;
}
// For both VP8 and VP9, we only look for resolution changes
// on keyframes. Other resolution changes are invalid.
if (!info.mKeyFrame) {
return NS_OK;
}
nsresult rv = NS_OK;
if (mInfo) {
if (mInfo.ref().IsCompatible(info)) {
return rv;
}
// We can't properly determine the image rect once we've had a resolution
// change.
mCurrentConfig.ResetImageRect();
PROFILER_MARKER_TEXT(
"VPX Stream Change", MEDIA_PLAYBACK, {},
"VPXChangeMonitor::CheckForChange has detected a change in the "
"stream and will request a new decoder");
rv = NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER;
} else if (mCurrentConfig.mImage != info.mImage ||
mCurrentConfig.mDisplay != info.mDisplay) {
// We can't properly determine the image rect if we're changing
// resolution based on sample information.
mCurrentConfig.ResetImageRect();
PROFILER_MARKER_TEXT("VPX Stream Init Discrepancy", MEDIA_PLAYBACK, {},
"VPXChangeMonitor::CheckForChange has detected a "
"discrepancy between initialization data and stream "
"content and will request a new decoder");
rv = NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER;
}
LOG("Detect inband %s resolution changes, image (%" PRId32 ",%" PRId32
")->(%" PRId32 ",%" PRId32 "), display (%" PRId32 ",%" PRId32
")->(%" PRId32 ",%" PRId32 " %s)",
mCodec == VPXDecoder::Codec::VP9 ? "VP9" : "VP8",
mCurrentConfig.mImage.Width(), mCurrentConfig.mImage.Height(),
info.mImage.Width(), info.mImage.Height(),
mCurrentConfig.mDisplay.Width(), mCurrentConfig.mDisplay.Height(),
info.mDisplay.Width(), info.mDisplay.Height(),
info.mDisplayAndImageDifferent ? "specified" : "unspecified");
bool imageSizeEmpty = mCurrentConfig.mImage.IsEmpty();
mInfo = Some(info);
mCurrentConfig.mImage = info.mImage;
if (imageSizeEmpty || info.mDisplayAndImageDifferent) {
// If the flag to change the display size is set in the sequence, we
// set our original values to begin rescaling according to the new values.
mCurrentConfig.mDisplay = info.mDisplay;
mPixelAspectRatio = GetPixelAspectRatio(info.mImage, info.mDisplay);
} else {
mCurrentConfig.mDisplay =
ApplyPixelAspectRatio(mPixelAspectRatio, info.mImage);
}
mCurrentConfig.mColorDepth = gfx::ColorDepthForBitDepth(info.mBitDepth);
mCurrentConfig.mColorSpace = Some(info.ColorSpace());
// VPX bitstream doesn't specify color primaries.
// We don't update the transfer function here, because VPX bitstream
// doesn't specify the transfer function. Instead, we keep the transfer
// function (if any) that was set in mCurrentConfig when we were created.
// If a video changes colorspaces away from BT2020, we won't clear
// mTransferFunction, in case the video changes back to BT2020 and we
// need the value again.
mCurrentConfig.mColorRange = info.ColorRange();
if (mCodec == VPXDecoder::Codec::VP9) {
mCurrentConfig.mExtraData->ClearAndRetainStorage();
VPXDecoder::GetVPCCBox(mCurrentConfig.mExtraData, info);
}
mTrackInfo = new TrackInfoSharedPtr(mCurrentConfig, mStreamID++);
return rv;
}
const TrackInfo& Config() const override { return mCurrentConfig; }
MediaResult PrepareSample(MediaDataDecoder::ConversionRequired aConversion,
MediaRawData* aSample,
bool aNeedKeyFrame) override {
aSample->mTrackInfo = mTrackInfo;
return NS_OK;
}
private:
VideoInfo mCurrentConfig;
const VPXDecoder::Codec mCodec;
Maybe<VPXDecoder::VPXStreamInfo> mInfo;
uint32_t mStreamID = 0;
RefPtr<TrackInfoSharedPtr> mTrackInfo;
double mPixelAspectRatio;
};
#ifdef MOZ_AV1
class AV1ChangeMonitor : public MediaChangeMonitor::CodecChangeMonitor {
public:
explicit AV1ChangeMonitor(const VideoInfo& aInfo)
: mCurrentConfig(aInfo),
mPixelAspectRatio(GetPixelAspectRatio(aInfo.mImage, aInfo.mDisplay)) {
mTrackInfo = new TrackInfoSharedPtr(mCurrentConfig, mStreamID++);
if (mCurrentConfig.mExtraData && !mCurrentConfig.mExtraData->IsEmpty()) {
// If we're passed AV1 codec configuration, store it so that we can
// instantiate a decoder in MediaChangeMonitor::Create.
AOMDecoder::AV1SequenceInfo seqInfo;
MediaResult seqHdrResult;
AOMDecoder::TryReadAV1CBox(mCurrentConfig.mExtraData, seqInfo,
seqHdrResult);
// If the av1C box doesn't include a sequence header specifying image
// size, keep the one provided by VideoInfo.
if (seqHdrResult.Code() != NS_OK) {
seqInfo.mImage = mCurrentConfig.mImage;
}
UpdateConfig(seqInfo);
}
}
bool CanBeInstantiated() const override {
// We want to have enough codec configuration to determine whether hardware
// decoding can be used before creating a decoder. The av1C box or a
// sequence header from a sample will contain this information.
return mInfo || mCurrentConfig.mCrypto.IsEncrypted();
}
void UpdateConfig(const AOMDecoder::AV1SequenceInfo& aInfo) {
mInfo = Some(aInfo);
mCurrentConfig.mColorDepth = gfx::ColorDepthForBitDepth(aInfo.mBitDepth);
mCurrentConfig.mColorSpace = gfxUtils::CicpToColorSpace(
aInfo.mColorSpace.mMatrix, aInfo.mColorSpace.mPrimaries,
gMediaDecoderLog);
mCurrentConfig.mColorPrimaries = gfxUtils::CicpToColorPrimaries(
aInfo.mColorSpace.mPrimaries, gMediaDecoderLog);
mCurrentConfig.mTransferFunction =
gfxUtils::CicpToTransferFunction(aInfo.mColorSpace.mTransfer);
mCurrentConfig.mColorRange = aInfo.mColorSpace.mRange;
if (mCurrentConfig.mImage != mInfo->mImage) {
gfx::IntSize newDisplay =
ApplyPixelAspectRatio(mPixelAspectRatio, aInfo.mImage);
LOG("AV1ChangeMonitor detected a resolution change in-band, image "
"(%" PRIu32 ",%" PRIu32 ")->(%" PRIu32 ",%" PRIu32
"), display (%" PRIu32 ",%" PRIu32 ")->(%" PRIu32 ",%" PRIu32
" from PAR)",
mCurrentConfig.mImage.Width(), mCurrentConfig.mImage.Height(),
aInfo.mImage.Width(), aInfo.mImage.Height(),
mCurrentConfig.mDisplay.Width(), mCurrentConfig.mDisplay.Height(),
newDisplay.Width(), newDisplay.Height());
mCurrentConfig.mImage = aInfo.mImage;
mCurrentConfig.mDisplay = newDisplay;
mCurrentConfig.ResetImageRect();
}
bool wroteSequenceHeader = false;
// Our headers should all be around the same size.
mCurrentConfig.mExtraData->ClearAndRetainStorage();
AOMDecoder::WriteAV1CBox(aInfo, mCurrentConfig.mExtraData.get(),
wroteSequenceHeader);
// Header should always be written ReadSequenceHeaderInfo succeeds.
MOZ_ASSERT(wroteSequenceHeader);
}
MediaResult CheckForChange(MediaRawData* aSample) override {
// Don't look at encrypted content.
if (aSample->mCrypto.IsEncrypted()) {
return NS_OK;
}
auto dataSpan = Span<const uint8_t>(aSample->Data(), aSample->Size());
// We don't trust the keyframe flag as set on the MediaRawData.
AOMDecoder::AV1SequenceInfo info;
MediaResult seqHdrResult =
AOMDecoder::ReadSequenceHeaderInfo(dataSpan, info);
nsresult seqHdrCode = seqHdrResult.Code();
if (seqHdrCode == NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA) {
return NS_OK;
}
if (seqHdrCode != NS_OK) {
LOG("AV1ChangeMonitor::CheckForChange read a corrupted sample: %s",
seqHdrResult.Description().get());
return seqHdrResult;
}
nsresult rv = NS_OK;
if (mInfo.isSome() &&
(mInfo->mProfile != info.mProfile ||
mInfo->ColorDepth() != info.ColorDepth() ||
mInfo->mMonochrome != info.mMonochrome ||
mInfo->mSubsamplingX != info.mSubsamplingX ||
mInfo->mSubsamplingY != info.mSubsamplingY ||
mInfo->mChromaSamplePosition != info.mChromaSamplePosition ||
mInfo->mImage != info.mImage)) {
PROFILER_MARKER_TEXT(
"AV1 Stream Change", MEDIA_PLAYBACK, {},
"AV1ChangeMonitor::CheckForChange has detected a change in a "
"stream and will request a new decoder");
LOG("AV1ChangeMonitor detected a change and requests a new decoder");
rv = NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER;
}
UpdateConfig(info);
if (rv == NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER) {
mTrackInfo = new TrackInfoSharedPtr(mCurrentConfig, mStreamID++);
}
return rv;
}
const TrackInfo& Config() const override { return mCurrentConfig; }
MediaResult PrepareSample(MediaDataDecoder::ConversionRequired aConversion,
MediaRawData* aSample,
bool aNeedKeyFrame) override {
aSample->mTrackInfo = mTrackInfo;
return NS_OK;
}
private:
VideoInfo mCurrentConfig;
Maybe<AOMDecoder::AV1SequenceInfo> mInfo;
uint32_t mStreamID = 0;
RefPtr<TrackInfoSharedPtr> mTrackInfo;
double mPixelAspectRatio;
};
#endif
MediaChangeMonitor::MediaChangeMonitor(
PDMFactory* aPDMFactory,
UniquePtr<CodecChangeMonitor>&& aCodecChangeMonitor,
MediaDataDecoder* aDecoder, const CreateDecoderParams& aParams)
: mChangeMonitor(std::move(aCodecChangeMonitor)),
mPDMFactory(aPDMFactory),
mCurrentConfig(aParams.VideoConfig()),
mDecoder(aDecoder),
mParams(aParams) {}
/* static */
RefPtr<PlatformDecoderModule::CreateDecoderPromise> MediaChangeMonitor::Create(
PDMFactory* aPDMFactory, const CreateDecoderParams& aParams) {
UniquePtr<CodecChangeMonitor> changeMonitor;
const VideoInfo& currentConfig = aParams.VideoConfig();
if (VPXDecoder::IsVPX(currentConfig.mMimeType)) {
changeMonitor = MakeUnique<VPXChangeMonitor>(currentConfig);
#ifdef MOZ_AV1
} else if (AOMDecoder::IsAV1(currentConfig.mMimeType)) {
changeMonitor = MakeUnique<AV1ChangeMonitor>(currentConfig);
#endif
} else if (MP4Decoder::IsHEVC(currentConfig.mMimeType)) {
changeMonitor = MakeUnique<HEVCChangeMonitor>(currentConfig);
} else {
MOZ_ASSERT(MP4Decoder::IsH264(currentConfig.mMimeType));
changeMonitor = MakeUnique<H264ChangeMonitor>(
currentConfig, aParams.mOptions.contains(
CreateDecoderParams::Option::FullH264Parsing));
}
// The change monitor may have an updated track config. E.g. the h264 monitor
// may update the config after parsing extra data in the VideoInfo. Create a
// new set of params with the updated track info from our monitor and the
// other params for aParams and use that going forward.
const CreateDecoderParams updatedParams{changeMonitor->Config(), aParams};
RefPtr<MediaChangeMonitor> instance = new MediaChangeMonitor(
aPDMFactory, std::move(changeMonitor), nullptr, updatedParams);
if (instance->mChangeMonitor->CanBeInstantiated()) {
RefPtr<PlatformDecoderModule::CreateDecoderPromise> p =
instance->CreateDecoder()->Then(
GetCurrentSerialEventTarget(), __func__,
[instance = RefPtr{instance}] {
return PlatformDecoderModule::CreateDecoderPromise::
CreateAndResolve(instance, __func__);
},
[](const MediaResult& aError) {
return PlatformDecoderModule::CreateDecoderPromise::
CreateAndReject(aError, __func__);
});
return p;
}
return PlatformDecoderModule::CreateDecoderPromise::CreateAndResolve(
instance, __func__);
}
MediaChangeMonitor::~MediaChangeMonitor() = default;
RefPtr<MediaDataDecoder::InitPromise> MediaChangeMonitor::Init() {
mThread = GetCurrentSerialEventTarget();
if (mDecoder) {
RefPtr<InitPromise> p = mInitPromise.Ensure(__func__);
RefPtr<MediaChangeMonitor> self = this;
mDecoder->Init()
->Then(GetCurrentSerialEventTarget(), __func__,
[self, this](InitPromise::ResolveOrRejectValue&& aValue) {
mInitPromiseRequest.Complete();
if (aValue.IsResolve()) {
mDecoderInitialized = true;
mConversionRequired = Some(mDecoder->NeedsConversion());
mCanRecycleDecoder = Some(CanRecycleDecoder());
if (mPendingSeekThreshold) {
mDecoder->SetSeekThreshold(*mPendingSeekThreshold);
mPendingSeekThreshold.reset();
}
}
return mInitPromise.ResolveOrRejectIfExists(std::move(aValue),
__func__);
})
->Track(mInitPromiseRequest);
return p;
}
// We haven't been able to initialize a decoder due to missing
// extradata.
return MediaDataDecoder::InitPromise::CreateAndResolve(TrackType::kVideoTrack,
__func__);
}
RefPtr<MediaDataDecoder::DecodePromise> MediaChangeMonitor::Decode(
MediaRawData* aSample) {
AssertOnThread();
MOZ_RELEASE_ASSERT(mFlushPromise.IsEmpty(),
"Flush operation didn't complete");
MOZ_RELEASE_ASSERT(
!mDecodePromiseRequest.Exists() && !mInitPromiseRequest.Exists(),
"Can't request a new decode until previous one completed");
MediaResult rv = CheckForChange(aSample);
if (rv == NS_ERROR_NOT_INITIALIZED) {
// We are missing the required init data to create the decoder.
if (mParams.mOptions.contains(
CreateDecoderParams::Option::ErrorIfNoInitializationData)) {
// This frame can't be decoded and should be treated as an error.
return DecodePromise::CreateAndReject(rv, __func__);
}
// Swallow the frame, and await delivery of init data.
return DecodePromise::CreateAndResolve(DecodedData(), __func__);
}
if (rv == NS_ERROR_DOM_MEDIA_INITIALIZING_DECODER) {
// The decoder is pending initialization.
RefPtr<DecodePromise> p = mDecodePromise.Ensure(__func__);
return p;
}
if (NS_FAILED(rv)) {
return DecodePromise::CreateAndReject(rv, __func__);
}
if (mNeedKeyframe && !aSample->mKeyframe) {
return DecodePromise::CreateAndResolve(DecodedData(), __func__);
}
rv = mChangeMonitor->PrepareSample(*mConversionRequired, aSample,
mNeedKeyframe);
if (NS_FAILED(rv)) {
return DecodePromise::CreateAndReject(rv, __func__);
}
mNeedKeyframe = false;
return mDecoder->Decode(aSample);
}
RefPtr<MediaDataDecoder::FlushPromise> MediaChangeMonitor::Flush() {
AssertOnThread();
mDecodePromiseRequest.DisconnectIfExists();
mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
mNeedKeyframe = true;
mPendingFrames.Clear();
MOZ_RELEASE_ASSERT(mFlushPromise.IsEmpty(), "Previous flush didn't complete");
/*
When we detect a change of content in the byte stream, we first drain the
current decoder (1), flush (2), shut it down (3) create a new decoder (4)
and initialize it (5). It is possible for MediaChangeMonitor::Flush to be
called during any of those times. If during (1):
- mDrainRequest will not be empty.
- The old decoder can still be used, with the current extradata as
stored in mCurrentConfig.mExtraData.
If during (2):
- mFlushRequest will not be empty.
- The old decoder can still be used, with the current extradata as
stored in mCurrentConfig.mExtraData.
If during (3):
- mShutdownRequest won't be empty.
- mDecoder is empty.
- The old decoder is no longer referenced by the MediaChangeMonitor.
If during (4):
- mDecoderRequest won't be empty.
- mDecoder is not set. Steps will continue to (5) to set and initialize it
If during (5):
- mInitPromiseRequest won't be empty.
- mDecoder is set but not usable yet.
*/
if (mDrainRequest.Exists() || mFlushRequest.Exists() ||
mShutdownRequest.Exists() || mDecoderRequest.Exists() ||
mInitPromiseRequest.Exists()) {
// We let the current decoder complete and will resume after.
RefPtr<FlushPromise> p = mFlushPromise.Ensure(__func__);
return p;
}
if (mDecoder && mDecoderInitialized) {
return mDecoder->Flush();
}
return FlushPromise::CreateAndResolve(true, __func__);
}
RefPtr<MediaDataDecoder::DecodePromise> MediaChangeMonitor::Drain() {
AssertOnThread();
MOZ_RELEASE_ASSERT(!mDrainRequest.Exists());
mNeedKeyframe = true;
if (mDecoder) {
return mDecoder->Drain();
}
return DecodePromise::CreateAndResolve(DecodedData(), __func__);
}
RefPtr<ShutdownPromise> MediaChangeMonitor::Shutdown() {
AssertOnThread();
mInitPromiseRequest.DisconnectIfExists();
mInitPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
mDecodePromiseRequest.DisconnectIfExists();
mDecodePromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
mDrainRequest.DisconnectIfExists();
mFlushRequest.DisconnectIfExists();
mFlushPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
mShutdownRequest.DisconnectIfExists();
if (mShutdownPromise) {
// We have a shutdown in progress, return that promise instead as we can't
// shutdown a decoder twice.
RefPtr<ShutdownPromise> p = std::move(mShutdownPromise);
return p;
}
return ShutdownDecoder();
}
RefPtr<ShutdownPromise> MediaChangeMonitor::ShutdownDecoder() {
AssertOnThread();
mConversionRequired.reset();
if (mDecoder) {
MutexAutoLock lock(mMutex);
RefPtr<MediaDataDecoder> decoder = std::move(mDecoder);
return decoder->Shutdown();
}
return ShutdownPromise::CreateAndResolve(true, __func__);
}
bool MediaChangeMonitor::IsHardwareAccelerated(
nsACString& aFailureReason) const {
if (mDecoder) {
return mDecoder->IsHardwareAccelerated(aFailureReason);
}
#ifdef MOZ_APPLEMEDIA
// On mac, we can assume H264 is hardware accelerated for now.
// This allows MediaCapabilities to report that playback will be smooth.
// Which will always be.
return true;
#else
return mChangeMonitor->IsHardwareAccelerated(aFailureReason);
#endif
}
void MediaChangeMonitor::SetSeekThreshold(const media::TimeUnit& aTime) {
GetCurrentSerialEventTarget()->Dispatch(NS_NewRunnableFunction(
"MediaChangeMonitor::SetSeekThreshold",
[self = RefPtr<MediaChangeMonitor>(this), time = aTime, this] {
// During the shutdown.
if (mShutdownPromise) {
return;
}
if (mDecoder && mDecoderInitialized) {
mDecoder->SetSeekThreshold(time);
} else {
mPendingSeekThreshold = Some(time);
}
}));
}
RefPtr<MediaChangeMonitor::CreateDecoderPromise>
MediaChangeMonitor::CreateDecoder() {
mCurrentConfig = *mChangeMonitor->Config().GetAsVideoInfo();
RefPtr<CreateDecoderPromise> p =
mPDMFactory
->CreateDecoder(
{mCurrentConfig, mParams, CreateDecoderParams::NoWrapper(true)})
->Then(
GetCurrentSerialEventTarget(), __func__,
[self = RefPtr{this}, this](RefPtr<MediaDataDecoder>&& aDecoder) {
MutexAutoLock lock(mMutex);
mDecoder = std::move(aDecoder);
DDLINKCHILD("decoder", mDecoder.get());
return CreateDecoderPromise::CreateAndResolve(true, __func__);
},
[self = RefPtr{this}](const MediaResult& aError) {
return CreateDecoderPromise::CreateAndReject(aError, __func__);
});
mDecoderInitialized = false;
mNeedKeyframe = true;
return p;
}
MediaResult MediaChangeMonitor::CreateDecoderAndInit(MediaRawData* aSample) {
MOZ_ASSERT(mThread && mThread->IsOnCurrentThread());
MediaResult rv = mChangeMonitor->CheckForChange(aSample);
if (!NS_SUCCEEDED(rv) && rv != NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER) {
return rv;
}
if (!mChangeMonitor->CanBeInstantiated()) {
// Nothing found yet, will try again later.
return NS_ERROR_NOT_INITIALIZED;
}
CreateDecoder()
->Then(
GetCurrentSerialEventTarget(), __func__,
[self = RefPtr{this}, this, sample = RefPtr{aSample}] {
mDecoderRequest.Complete();
mDecoder->Init()
->Then(
GetCurrentSerialEventTarget(), __func__,
[self, sample, this](const TrackType aTrackType) {
mInitPromiseRequest.Complete();
mDecoderInitialized = true;
mConversionRequired = Some(mDecoder->NeedsConversion());
mCanRecycleDecoder = Some(CanRecycleDecoder());
if (mPendingSeekThreshold) {
mDecoder->SetSeekThreshold(*mPendingSeekThreshold);
mPendingSeekThreshold.reset();
}
if (!mFlushPromise.IsEmpty()) {
// A Flush is pending, abort the current operation.
mFlushPromise.Resolve(true, __func__);
return;
}
DecodeFirstSample(sample);
},
[self, this](const MediaResult& aError) {
mInitPromiseRequest.Complete();
if (!mFlushPromise.IsEmpty()) {
// A Flush is pending, abort the current operation.
mFlushPromise.Reject(aError, __func__);
return;
}
mDecodePromise.Reject(
MediaResult(
aError.Code(),
RESULT_DETAIL("Unable to initialize decoder")),
__func__);
})
->Track(mInitPromiseRequest);
},
[self = RefPtr{this}, this](const MediaResult& aError) {
mDecoderRequest.Complete();
if (!mFlushPromise.IsEmpty()) {
// A Flush is pending, abort the current operation.
mFlushPromise.Reject(aError, __func__);
return;
}
mDecodePromise.Reject(
MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
RESULT_DETAIL("Unable to create decoder")),
__func__);
})
->Track(mDecoderRequest);
return NS_ERROR_DOM_MEDIA_INITIALIZING_DECODER;
}
bool MediaChangeMonitor::CanRecycleDecoder() const {
MOZ_ASSERT(mDecoder);
return StaticPrefs::media_decoder_recycle_enabled() &&
mDecoder->SupportDecoderRecycling();
}
void MediaChangeMonitor::DecodeFirstSample(MediaRawData* aSample) {
// We feed all the data to AnnexB decoder as a non-keyframe could contain
// the SPS/PPS when used with WebRTC and this data is needed by the decoder.
if (mNeedKeyframe && !aSample->mKeyframe &&
*mConversionRequired != ConversionRequired::kNeedAnnexB) {
mDecodePromise.Resolve(std::move(mPendingFrames), __func__);
mPendingFrames = DecodedData();
return;
}
MediaResult rv = mChangeMonitor->PrepareSample(*mConversionRequired, aSample,
mNeedKeyframe);
if (NS_FAILED(rv)) {
mDecodePromise.Reject(rv, __func__);
return;
}
if (aSample->mKeyframe) {
mNeedKeyframe = false;
}
RefPtr<MediaChangeMonitor> self = this;
mDecoder->Decode(aSample)
->Then(
GetCurrentSerialEventTarget(), __func__,
[self, this](MediaDataDecoder::DecodedData&& aResults) {
mDecodePromiseRequest.Complete();
mPendingFrames.AppendElements(std::move(aResults));
mDecodePromise.Resolve(std::move(mPendingFrames), __func__);
mPendingFrames = DecodedData();
},
[self, this](const MediaResult& aError) {
mDecodePromiseRequest.Complete();
mDecodePromise.Reject(aError, __func__);
})
->Track(mDecodePromiseRequest);
}
MediaResult MediaChangeMonitor::CheckForChange(MediaRawData* aSample) {
if (!mDecoder) {
return CreateDecoderAndInit(aSample);
}
MediaResult rv = mChangeMonitor->CheckForChange(aSample);
if (NS_SUCCEEDED(rv) || rv != NS_ERROR_DOM_MEDIA_NEED_NEW_DECODER) {
return rv;
}
if (*mCanRecycleDecoder) {
// Do not recreate the decoder, reuse it.
mNeedKeyframe = true;
return NS_OK;
}
// The content has changed, signal to drain the current decoder and once done
// create a new one.
DrainThenFlushDecoder(aSample);
return NS_ERROR_DOM_MEDIA_INITIALIZING_DECODER;
}
void MediaChangeMonitor::DrainThenFlushDecoder(MediaRawData* aPendingSample) {
AssertOnThread();
MOZ_ASSERT(mDecoderInitialized);
RefPtr<MediaRawData> sample = aPendingSample;
RefPtr<MediaChangeMonitor> self = this;
mDecoder->Drain()
->Then(
GetCurrentSerialEventTarget(), __func__,
[self, sample, this](MediaDataDecoder::DecodedData&& aResults) {
mDrainRequest.Complete();
if (!mFlushPromise.IsEmpty()) {
// A Flush is pending, abort the current operation.
mFlushPromise.Resolve(true, __func__);
return;
}
if (aResults.Length() > 0) {
mPendingFrames.AppendElements(std::move(aResults));
DrainThenFlushDecoder(sample);
return;
}
// We've completed the draining operation, we can now flush the
// decoder.
FlushThenShutdownDecoder(sample);
},
[self, this](const MediaResult& aError) {
mDrainRequest.Complete();
if (!mFlushPromise.IsEmpty()) {
// A Flush is pending, abort the current operation.
mFlushPromise.Reject(aError, __func__);
return;
}
mDecodePromise.Reject(aError, __func__);
})
->Track(mDrainRequest);
}
void MediaChangeMonitor::FlushThenShutdownDecoder(
MediaRawData* aPendingSample) {
AssertOnThread();
MOZ_ASSERT(mDecoderInitialized);
RefPtr<MediaRawData> sample = aPendingSample;
RefPtr<MediaChangeMonitor> self = this;
mDecoder->Flush()
->Then(
GetCurrentSerialEventTarget(), __func__,
[self, sample, this]() {
mFlushRequest.Complete();
if (!mFlushPromise.IsEmpty()) {
// A Flush is pending, abort the current operation.
mFlushPromise.Resolve(true, __func__);
return;
}
mShutdownPromise = ShutdownDecoder();
mShutdownPromise
->Then(
GetCurrentSerialEventTarget(), __func__,
[self, sample, this]() {
mShutdownRequest.Complete();
mShutdownPromise = nullptr;
if (!mFlushPromise.IsEmpty()) {
// A Flush is pending, abort the current
// operation.
mFlushPromise.Resolve(true, __func__);
return;
}
MediaResult rv = CreateDecoderAndInit(sample);
if (rv == NS_ERROR_DOM_MEDIA_INITIALIZING_DECODER) {
// All good so far, will continue later.
return;
}
MOZ_ASSERT(NS_FAILED(rv));
mDecodePromise.Reject(rv, __func__);
return;
},
[] { MOZ_CRASH("Can't reach here'"); })
->Track(mShutdownRequest);
},
[self, this](const MediaResult& aError) {
mFlushRequest.Complete();
if (!mFlushPromise.IsEmpty()) {
// A Flush is pending, abort the current operation.
mFlushPromise.Reject(aError, __func__);
return;
}
mDecodePromise.Reject(aError, __func__);
})
->Track(mFlushRequest);
}
MediaDataDecoder* MediaChangeMonitor::GetDecoderOnNonOwnerThread() const {
MutexAutoLock lock(mMutex);
return mDecoder;
}
#undef LOG
} // namespace mozilla