Source code

Revision control

Other Tools

/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set ts=2 sw=2 sts=2 et cindent: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "MediaFormatReader.h"
#include <algorithm>
#include <map>
#include <queue>
#include "AllocationPolicy.h"
#include "DecoderBenchmark.h"
#include "GeckoProfiler.h"
#include "MediaData.h"
#include "MediaDataDecoderProxy.h"
#include "MediaInfo.h"
#include "VideoFrameContainer.h"
#include "VideoUtils.h"
#include "mozilla/AbstractThread.h"
#include "mozilla/CDMProxy.h"
#include "mozilla/ClearOnShutdown.h"
#include "mozilla/NotNull.h"
#include "mozilla/Preferences.h"
#include "mozilla/SharedThreadPool.h"
#include "mozilla/StaticPrefs_media.h"
#include "mozilla/TaskQueue.h"
#include "mozilla/Unused.h"
#include "nsContentUtils.h"
#include "nsPrintfCString.h"
using namespace mozilla::media;
static mozilla::LazyLogModule sFormatDecoderLog("MediaFormatReader");
mozilla::LazyLogModule gMediaDemuxerLog("MediaDemuxer");
#define LOG(arg, ...) \
DDMOZ_LOG(sFormatDecoderLog, mozilla::LogLevel::Debug, "::%s: " arg, \
__func__, ##__VA_ARGS__)
#define LOGV(arg, ...) \
DDMOZ_LOG(sFormatDecoderLog, mozilla::LogLevel::Verbose, "::%s: " arg, \
__func__, ##__VA_ARGS__)
#define NS_DispatchToMainThread(...) CompileError_UseAbstractMainThreadInstead
namespace mozilla {
typedef void* MediaDataDecoderID;
/**
* This class tracks shutdown promises to ensure all decoders are shut down
* completely before MFR continues the rest of the shutdown procedure.
*/
class MediaFormatReader::ShutdownPromisePool {
public:
ShutdownPromisePool()
: mOnShutdownComplete(new ShutdownPromise::Private(__func__)) {}
// Return a promise which will be resolved when all the tracking promises
// are resolved. Note no more promises should be added for tracking once
// this function is called.
RefPtr<ShutdownPromise> Shutdown();
// Track a shutdown promise.
void Track(RefPtr<ShutdownPromise> aPromise);
// Shut down a decoder and track its shutdown promise.
void ShutdownDecoder(already_AddRefed<MediaDataDecoder> aDecoder) {
Track(RefPtr<MediaDataDecoder>(aDecoder)->Shutdown());
}
private:
bool mShutdown = false;
const RefPtr<ShutdownPromise::Private> mOnShutdownComplete;
nsTHashtable<nsRefPtrHashKey<ShutdownPromise>> mPromises;
};
RefPtr<ShutdownPromise> MediaFormatReader::ShutdownPromisePool::Shutdown() {
MOZ_DIAGNOSTIC_ASSERT(!mShutdown);
mShutdown = true;
if (mPromises.Count() == 0) {
mOnShutdownComplete->Resolve(true, __func__);
}
return mOnShutdownComplete;
}
void MediaFormatReader::ShutdownPromisePool::Track(
RefPtr<ShutdownPromise> aPromise) {
MOZ_DIAGNOSTIC_ASSERT(!mShutdown);
MOZ_DIAGNOSTIC_ASSERT(!mPromises.Contains(aPromise));
mPromises.PutEntry(aPromise);
aPromise->Then(AbstractThread::GetCurrent(), __func__, [aPromise, this]() {
MOZ_DIAGNOSTIC_ASSERT(mPromises.Contains(aPromise));
mPromises.RemoveEntry(aPromise);
if (mShutdown && mPromises.Count() == 0) {
mOnShutdownComplete->Resolve(true, __func__);
}
});
}
void MediaFormatReader::DecoderData::ShutdownDecoder() {
MOZ_ASSERT(mOwner->OnTaskQueue());
MutexAutoLock lock(mMutex);
if (!mDecoder) {
// No decoder to shut down.
return;
}
if (mFlushing) {
// Flush is is in action. Shutdown will be initiated after flush completes.
MOZ_DIAGNOSTIC_ASSERT(mShutdownPromise);
mOwner->mShutdownPromisePool->Track(mShutdownPromise->Ensure(__func__));
// The order of decoder creation and shutdown is handled by LocalAllocPolicy
// and ShutdownPromisePool. MFR can now reset these members to a fresh state
// and be ready to create new decoders again without explicitly waiting for
// flush/shutdown to complete.
mShutdownPromise = nullptr;
mFlushing = false;
} else {
// No flush is in action. We can shut down the decoder now.
mOwner->mShutdownPromisePool->Track(mDecoder->Shutdown());
}
// mShutdownPromisePool will handle the order of decoder shutdown so
// we can forget mDecoder and be ready to create a new one.
mDecoder = nullptr;
mDescription = "shutdown"_ns;
mOwner->ScheduleUpdate(mType == MediaData::Type::AUDIO_DATA
? TrackType::kAudioTrack
: TrackType::kVideoTrack);
}
void MediaFormatReader::DecoderData::Flush() {
AUTO_PROFILER_LABEL("MediaFormatReader::Flush", MEDIA_PLAYBACK);
MOZ_ASSERT(mOwner->OnTaskQueue());
if (mFlushing || mFlushed) {
// Flush still pending or already flushed, nothing more to do.
return;
}
mDecodeRequest.DisconnectIfExists();
mDrainRequest.DisconnectIfExists();
mDrainState = DrainState::None;
CancelWaitingForKey();
mOutput.Clear();
mNumSamplesInput = 0;
mNumSamplesOutput = 0;
mSizeOfQueue = 0;
if (mDecoder) {
TrackType type = mType == MediaData::Type::AUDIO_DATA
? TrackType::kAudioTrack
: TrackType::kVideoTrack;
mFlushing = true;
MOZ_DIAGNOSTIC_ASSERT(!mShutdownPromise);
mShutdownPromise = new SharedShutdownPromiseHolder();
RefPtr<SharedShutdownPromiseHolder> p = mShutdownPromise;
RefPtr<MediaDataDecoder> d = mDecoder;
DDLOGEX2("MediaFormatReader::DecoderData", this, DDLogCategory::Log,
"flushing", DDNoValue{});
mDecoder->Flush()->Then(
mOwner->OwnerThread(), __func__,
[type, this, p, d]() {
AUTO_PROFILER_LABEL("MediaFormatReader::Flush:Resolved",
MEDIA_PLAYBACK);
DDLOGEX2("MediaFormatReader::DecoderData", this, DDLogCategory::Log,
"flushed", DDNoValue{});
if (!p->IsEmpty()) {
// Shutdown happened before flush completes.
// Let's continue to shut down the decoder. Note
// we don't access |this| because this decoder
// is no longer managed by MFR::DecoderData.
d->Shutdown()->ChainTo(p->Steal(), __func__);
return;
}
mFlushing = false;
mShutdownPromise = nullptr;
mOwner->ScheduleUpdate(type);
},
[type, this, p, d](const MediaResult& aError) {
AUTO_PROFILER_LABEL("MediaFormatReader::Flush:Rejected",
MEDIA_PLAYBACK);
DDLOGEX2("MediaFormatReader::DecoderData", this, DDLogCategory::Log,
"flush_error", aError);
if (!p->IsEmpty()) {
d->Shutdown()->ChainTo(p->Steal(), __func__);
return;
}
mFlushing = false;
mShutdownPromise = nullptr;
mOwner->NotifyError(type, aError);
});
}
mFlushed = true;
}
class MediaFormatReader::DecoderFactory {
using InitPromise = MediaDataDecoder::InitPromise;
using TokenPromise = AllocPolicy::Promise;
using Token = AllocPolicy::Token;
public:
explicit DecoderFactory(MediaFormatReader* aOwner)
: mAudio(aOwner->mAudio, TrackInfo::kAudioTrack, aOwner->OwnerThread()),
mVideo(aOwner->mVideo, TrackInfo::kVideoTrack, aOwner->OwnerThread()),
mOwner(WrapNotNull(aOwner)) {
DecoderDoctorLogger::LogConstruction("MediaFormatReader::DecoderFactory",
this);
DecoderDoctorLogger::LinkParentAndChild(
aOwner, "decoder factory", "MediaFormatReader::DecoderFactory", this);
}
~DecoderFactory() {
DecoderDoctorLogger::LogDestruction("MediaFormatReader::DecoderFactory",
this);
}
void CreateDecoder(TrackType aTrack);
// Shutdown any decoder pending initialization and reset mAudio/mVideo to its
// pristine state so CreateDecoder() is ready to be called again immediately.
void ShutdownDecoder(TrackType aTrack) {
MOZ_ASSERT(aTrack == TrackInfo::kAudioTrack ||
aTrack == TrackInfo::kVideoTrack);
auto& data = aTrack == TrackInfo::kAudioTrack ? mAudio : mVideo;
data.mPolicy->Cancel();
data.mTokenRequest.DisconnectIfExists();
data.mInitRequest.DisconnectIfExists();
if (data.mDecoder) {
mOwner->mShutdownPromisePool->ShutdownDecoder(data.mDecoder.forget());
}
data.mStage = Stage::None;
MOZ_ASSERT(!data.mToken);
}
private:
enum class Stage : int8_t { None, WaitForToken, CreateDecoder, WaitForInit };
struct Data {
Data(DecoderData& aOwnerData, TrackType aTrack, TaskQueue* aThread)
: mOwnerData(aOwnerData),
mTrack(aTrack),
mPolicy(new SingleAllocPolicy(aTrack, aThread)) {}
DecoderData& mOwnerData;
const TrackType mTrack;
RefPtr<SingleAllocPolicy> mPolicy;
Stage mStage = Stage::None;
RefPtr<Token> mToken;
RefPtr<MediaDataDecoder> mDecoder;
MozPromiseRequestHolder<TokenPromise> mTokenRequest;
MozPromiseRequestHolder<InitPromise> mInitRequest;
} mAudio, mVideo;
void RunStage(Data& aData);
MediaResult DoCreateDecoder(Data& aData);
void DoInitDecoder(Data& aData);
// guaranteed to be valid by the owner.
const NotNull<MediaFormatReader*> mOwner;
};
void MediaFormatReader::DecoderFactory::CreateDecoder(TrackType aTrack) {
MOZ_ASSERT(aTrack == TrackInfo::kAudioTrack ||
aTrack == TrackInfo::kVideoTrack);
RunStage(aTrack == TrackInfo::kAudioTrack ? mAudio : mVideo);
}
void MediaFormatReader::DecoderFactory::RunStage(Data& aData) {
switch (aData.mStage) {
case Stage::None: {
MOZ_ASSERT(!aData.mToken);
aData.mPolicy->Alloc()
->Then(
mOwner->OwnerThread(), __func__,
[this, &aData](RefPtr<Token> aToken) {
aData.mTokenRequest.Complete();
aData.mToken = std::move(aToken);
aData.mStage = Stage::CreateDecoder;
RunStage(aData);
},
[&aData]() {
aData.mTokenRequest.Complete();
aData.mStage = Stage::None;
})
->Track(aData.mTokenRequest);
aData.mStage = Stage::WaitForToken;
break;
}
case Stage::WaitForToken: {
MOZ_ASSERT(!aData.mToken);
MOZ_ASSERT(aData.mTokenRequest.Exists());
break;
}
case Stage::CreateDecoder: {
MOZ_ASSERT(aData.mToken);
MOZ_ASSERT(!aData.mDecoder);
MOZ_ASSERT(!aData.mInitRequest.Exists());
MediaResult rv = DoCreateDecoder(aData);
if (NS_FAILED(rv)) {
NS_WARNING("Error constructing decoders");
aData.mToken = nullptr;
aData.mStage = Stage::None;
aData.mOwnerData.mDescription = rv.Description();
DDLOGEX2("MediaFormatReader::DecoderFactory", this, DDLogCategory::Log,
"create_decoder_error", rv);
mOwner->NotifyError(aData.mTrack, rv);
return;
}
aData.mDecoder =
new AllocationWrapper(aData.mDecoder.forget(), aData.mToken.forget());
DecoderDoctorLogger::LinkParentAndChild(
aData.mDecoder.get(), "decoder", "MediaFormatReader::DecoderFactory",
this);
DoInitDecoder(aData);
aData.mStage = Stage::WaitForInit;
break;
}
case Stage::WaitForInit: {
MOZ_ASSERT(aData.mDecoder);
MOZ_ASSERT(aData.mInitRequest.Exists());
break;
}
}
}
MediaResult MediaFormatReader::DecoderFactory::DoCreateDecoder(Data& aData) {
AUTO_PROFILER_LABEL("DecoderFactory::DoCreateDecoder", MEDIA_PLAYBACK);
auto& ownerData = aData.mOwnerData;
auto& decoder = mOwner->GetDecoderData(aData.mTrack);
auto& platform =
decoder.IsEncrypted() ? mOwner->mEncryptedPlatform : mOwner->mPlatform;
if (!platform) {
platform = new PDMFactory();
if (decoder.IsEncrypted()) {
MOZ_ASSERT(mOwner->mCDMProxy);
platform->SetCDMProxy(mOwner->mCDMProxy);
}
}
// result may not be updated by PDMFactory::CreateDecoder, as such it must be
// initialized to a fatal error by default.
MediaResult result =
MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
nsPrintfCString("error creating %s decoder",
TrackTypeToStr(aData.mTrack)));
switch (aData.mTrack) {
case TrackInfo::kAudioTrack: {
RefPtr<MediaDataDecoder> decoder = platform->CreateDecoder(
{*ownerData.GetCurrentInfo()->GetAsAudioInfo(), mOwner->mCrashHelper,
CreateDecoderParams::UseNullDecoder(ownerData.mIsNullDecode),
&result, TrackInfo::kAudioTrack,
&mOwner->OnTrackWaitingForKeyProducer()});
if (!decoder) {
aData.mDecoder = nullptr;
break;
}
aData.mDecoder = new MediaDataDecoderProxy(
decoder.forget(), do_AddRef(ownerData.mTaskQueue.get()));
break;
}
case TrackType::kVideoTrack: {
// Decoders use the layers backend to decide if they can use hardware
// decoding, so specify LAYERS_NONE if we want to forcibly disable it.
using Option = CreateDecoderParams::Option;
using OptionSet = CreateDecoderParams::OptionSet;
RefPtr<MediaDataDecoder> decoder = platform->CreateDecoder(
{*ownerData.GetCurrentInfo()->GetAsVideoInfo(),
mOwner->mKnowsCompositor, mOwner->GetImageContainer(),
mOwner->mCrashHelper,
CreateDecoderParams::UseNullDecoder(ownerData.mIsNullDecode),
&result, TrackType::kVideoTrack,
&mOwner->OnTrackWaitingForKeyProducer(),
CreateDecoderParams::VideoFrameRate(ownerData.mMeanRate.Mean()),
OptionSet(ownerData.mHardwareDecodingDisabled
? Option::HardwareDecoderNotAllowed
: Option::Default)});
if (!decoder) {
aData.mDecoder = nullptr;
break;
}
aData.mDecoder = new MediaDataDecoderProxy(
decoder.forget(), do_AddRef(ownerData.mTaskQueue.get()));
break;
}
default:
break;
}
if (aData.mDecoder) {
return NS_OK;
}
MOZ_RELEASE_ASSERT(NS_FAILED(result), "PDM returned an invalid error code");
return result;
}
void MediaFormatReader::DecoderFactory::DoInitDecoder(Data& aData) {
AUTO_PROFILER_LABEL("DecoderFactory::DoInitDecoder", MEDIA_PLAYBACK);
auto& ownerData = aData.mOwnerData;
DDLOGEX2("MediaFormatReader::DecoderFactory", this, DDLogCategory::Log,
"initialize_decoder", DDNoValue{});
aData.mDecoder->Init()
->Then(
mOwner->OwnerThread(), __func__,
[this, &aData, &ownerData](TrackType aTrack) {
AUTO_PROFILER_LABEL("DecoderFactory::DoInitDecoder:Resolved",
MEDIA_PLAYBACK);
aData.mInitRequest.Complete();
aData.mStage = Stage::None;
MutexAutoLock lock(ownerData.mMutex);
ownerData.mDecoder = std::move(aData.mDecoder);
ownerData.mDescription = ownerData.mDecoder->GetDescriptionName();
DDLOGEX2("MediaFormatReader::DecoderFactory", this,
DDLogCategory::Log, "decoder_initialized", DDNoValue{});
DecoderDoctorLogger::LinkParentAndChild(
"MediaFormatReader::DecoderData", &ownerData, "decoder",
ownerData.mDecoder.get());
mOwner->SetVideoDecodeThreshold();
mOwner->ScheduleUpdate(aTrack);
if (aTrack == TrackInfo::kVideoTrack) {
DecoderBenchmark::CheckVersion(
ownerData.GetCurrentInfo()->mMimeType);
}
},
[this, &aData, &ownerData](const MediaResult& aError) {
AUTO_PROFILER_LABEL("DecoderFactory::DoInitDecoder:Rejected",
MEDIA_PLAYBACK);
aData.mInitRequest.Complete();
MOZ_RELEASE_ASSERT(!ownerData.mDecoder,
"Can't have a decoder already set");
aData.mStage = Stage::None;
mOwner->mShutdownPromisePool->ShutdownDecoder(
aData.mDecoder.forget());
DDLOGEX2("MediaFormatReader::DecoderFactory", this,
DDLogCategory::Log, "initialize_decoder_error", aError);
mOwner->NotifyError(aData.mTrack, aError);
})
->Track(aData.mInitRequest);
}
// DemuxerProxy ensures that the original main demuxer is only ever accessed
// via its own dedicated task queue.
// This ensure that the reader's taskqueue will never blocked while a demuxer
// is itself blocked attempting to access the MediaCache or the MediaResource.
class MediaFormatReader::DemuxerProxy {
using TrackType = TrackInfo::TrackType;
class Wrapper;
public:
explicit DemuxerProxy(MediaDataDemuxer* aDemuxer)
: mTaskQueue(
new TaskQueue(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER),
"DemuxerProxy::mTaskQueue")),
mData(new Data(aDemuxer)) {
MOZ_COUNT_CTOR(DemuxerProxy);
}
MOZ_COUNTED_DTOR(DemuxerProxy)
RefPtr<ShutdownPromise> Shutdown() {
RefPtr<Data> data = std::move(mData);
return InvokeAsync(mTaskQueue, __func__, [data]() {
// We need to clear our reference to the demuxer now. So that in the event
// the init promise wasn't resolved, such as what can happen with the
// mediasource demuxer that is waiting on more data, it will force the
// init promise to be rejected.
data->mDemuxer = nullptr;
data->mAudioDemuxer = nullptr;
data->mVideoDemuxer = nullptr;
return ShutdownPromise::CreateAndResolve(true, __func__);
});
}
RefPtr<MediaDataDemuxer::InitPromise> Init();
Wrapper* GetTrackDemuxer(TrackType aTrack, uint32_t aTrackNumber) {
MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
switch (aTrack) {
case TrackInfo::kAudioTrack:
return mData->mAudioDemuxer;
case TrackInfo::kVideoTrack:
return mData->mVideoDemuxer;
default:
return nullptr;
}
}
uint32_t GetNumberTracks(TrackType aTrack) const {
MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
switch (aTrack) {
case TrackInfo::kAudioTrack:
return mData->mNumAudioTrack;
case TrackInfo::kVideoTrack:
return mData->mNumVideoTrack;
default:
return 0;
}
}
bool IsSeekable() const {
MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
return mData->mSeekable;
}
bool IsSeekableOnlyInBufferedRanges() const {
MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
return mData->mSeekableOnlyInBufferedRange;
}
UniquePtr<EncryptionInfo> GetCrypto() const {
MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
if (!mData->mCrypto) {
return nullptr;
}
auto crypto = MakeUnique<EncryptionInfo>();
*crypto = *mData->mCrypto;
return crypto;
}
RefPtr<NotifyDataArrivedPromise> NotifyDataArrived();
bool ShouldComputeStartTime() const {
MOZ_RELEASE_ASSERT(mData && mData->mInitDone);
return mData->mShouldComputeStartTime;
}
private:
const RefPtr<TaskQueue> mTaskQueue;
struct Data {
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Data)
explicit Data(MediaDataDemuxer* aDemuxer)
: mInitDone(false), mDemuxer(aDemuxer) {}
Atomic<bool> mInitDone;
// Only ever accessed over mTaskQueue once.
RefPtr<MediaDataDemuxer> mDemuxer;
// Only accessed once InitPromise has been resolved and immutable after.
// So we can safely access them without the use of the mutex.
uint32_t mNumAudioTrack = 0;
RefPtr<Wrapper> mAudioDemuxer;
uint32_t mNumVideoTrack = 0;
RefPtr<Wrapper> mVideoDemuxer;
bool mSeekable = false;
bool mSeekableOnlyInBufferedRange = false;
bool mShouldComputeStartTime = true;
UniquePtr<EncryptionInfo> mCrypto;
private:
~Data() = default;
};
RefPtr<Data> mData;
};
class MediaFormatReader::DemuxerProxy::Wrapper : public MediaTrackDemuxer {
public:
Wrapper(MediaTrackDemuxer* aTrackDemuxer, TaskQueue* aTaskQueue)
: mMutex("TrackDemuxer Mutex"),
mTaskQueue(aTaskQueue),
mGetSamplesMayBlock(aTrackDemuxer->GetSamplesMayBlock()),
mInfo(aTrackDemuxer->GetInfo()),
mTrackDemuxer(aTrackDemuxer) {
DecoderDoctorLogger::LogConstructionAndBase(
"MediaFormatReader::DemuxerProxy::Wrapper", this,
static_cast<const MediaTrackDemuxer*>(this));
DecoderDoctorLogger::LinkParentAndChild(
"MediaFormatReader::DemuxerProxy::Wrapper", this, "track demuxer",
aTrackDemuxer);
}
UniquePtr<TrackInfo> GetInfo() const override {
if (!mInfo) {
return nullptr;
}
return mInfo->Clone();
}
RefPtr<SeekPromise> Seek(const TimeUnit& aTime) override {
RefPtr<Wrapper> self = this;
return InvokeAsync(
mTaskQueue, __func__,
[self, aTime]() { return self->mTrackDemuxer->Seek(aTime); })
->Then(
mTaskQueue, __func__,
[self](const TimeUnit& aTime) {
self->UpdateRandomAccessPoint();
return SeekPromise::CreateAndResolve(aTime, __func__);
},
[self](const MediaResult& aError) {
self->UpdateRandomAccessPoint();
return SeekPromise::CreateAndReject(aError, __func__);
});
}
RefPtr<SamplesPromise> GetSamples(int32_t aNumSamples) override {
RefPtr<Wrapper> self = this;
return InvokeAsync(mTaskQueue, __func__,
[self, aNumSamples]() {
return self->mTrackDemuxer->GetSamples(aNumSamples);
})
->Then(
mTaskQueue, __func__,
[self](RefPtr<SamplesHolder> aSamples) {
self->UpdateRandomAccessPoint();
return SamplesPromise::CreateAndResolve(aSamples.forget(),
__func__);
},
[self](const MediaResult& aError) {
self->UpdateRandomAccessPoint();
return SamplesPromise::CreateAndReject(aError, __func__);
});
}
bool GetSamplesMayBlock() const override { return mGetSamplesMayBlock; }
void Reset() override {
RefPtr<Wrapper> self = this;
nsresult rv = mTaskQueue->Dispatch(NS_NewRunnableFunction(
"MediaFormatReader::DemuxerProxy::Wrapper::Reset",
[self]() { self->mTrackDemuxer->Reset(); }));
MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
Unused << rv;
}
nsresult GetNextRandomAccessPoint(TimeUnit* aTime) override {
MutexAutoLock lock(mMutex);
if (NS_SUCCEEDED(mNextRandomAccessPointResult)) {
*aTime = mNextRandomAccessPoint;
}
return mNextRandomAccessPointResult;
}
RefPtr<SkipAccessPointPromise> SkipToNextRandomAccessPoint(
const TimeUnit& aTimeThreshold) override {
RefPtr<Wrapper> self = this;
return InvokeAsync(
mTaskQueue, __func__,
[self, aTimeThreshold]() {
return self->mTrackDemuxer->SkipToNextRandomAccessPoint(
aTimeThreshold);
})
->Then(
mTaskQueue, __func__,
[self](uint32_t aVal) {
self->UpdateRandomAccessPoint();
return SkipAccessPointPromise::CreateAndResolve(aVal, __func__);
},
[self](const SkipFailureHolder& aError) {
self->UpdateRandomAccessPoint();
return SkipAccessPointPromise::CreateAndReject(aError, __func__);
});
}
TimeIntervals GetBuffered() override {
MutexAutoLock lock(mMutex);
return mBuffered;
}
void BreakCycles() override {}
private:
Mutex mMutex;
const RefPtr<TaskQueue> mTaskQueue;
const bool mGetSamplesMayBlock;
const UniquePtr<TrackInfo> mInfo;
// mTrackDemuxer is only ever accessed on demuxer's task queue.
RefPtr<MediaTrackDemuxer> mTrackDemuxer;
// All following members are protected by mMutex
nsresult mNextRandomAccessPointResult = NS_OK;
TimeUnit mNextRandomAccessPoint;
TimeIntervals mBuffered;
friend class DemuxerProxy;
~Wrapper() {
RefPtr<MediaTrackDemuxer> trackDemuxer = std::move(mTrackDemuxer);
nsresult rv = mTaskQueue->Dispatch(NS_NewRunnableFunction(
"MediaFormatReader::DemuxerProxy::Wrapper::~Wrapper",
[trackDemuxer]() { trackDemuxer->BreakCycles(); }));
MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
Unused << rv;
DecoderDoctorLogger::LogDestruction(
"MediaFormatReader::DemuxerProxy::Wrapper", this);
}
void UpdateRandomAccessPoint() {
MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn());
if (!mTrackDemuxer) {
// Detached.
return;
}
MutexAutoLock lock(mMutex);
mNextRandomAccessPointResult =
mTrackDemuxer->GetNextRandomAccessPoint(&mNextRandomAccessPoint);
}
void UpdateBuffered() {
MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn());
if (!mTrackDemuxer) {
// Detached.
return;
}
MutexAutoLock lock(mMutex);
mBuffered = mTrackDemuxer->GetBuffered();
}
};
RefPtr<MediaDataDemuxer::InitPromise> MediaFormatReader::DemuxerProxy::Init() {
AUTO_PROFILER_LABEL("DemuxerProxy::Init", MEDIA_PLAYBACK);
using InitPromise = MediaDataDemuxer::InitPromise;
RefPtr<Data> data = mData;
RefPtr<TaskQueue> taskQueue = mTaskQueue;
return InvokeAsync(mTaskQueue, __func__,
[data, taskQueue]() {
if (!data->mDemuxer) {
return InitPromise::CreateAndReject(
NS_ERROR_DOM_MEDIA_CANCELED, __func__);
}
return data->mDemuxer->Init();
})
->Then(
taskQueue, __func__,
[data, taskQueue]() {
AUTO_PROFILER_LABEL("DemuxerProxy::Init:Resolved", MEDIA_PLAYBACK);
if (!data->mDemuxer) { // Was shutdown.
return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED,
__func__);
}
data->mNumAudioTrack =
data->mDemuxer->GetNumberTracks(TrackInfo::kAudioTrack);
if (data->mNumAudioTrack) {
RefPtr<MediaTrackDemuxer> d =
data->mDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0);
if (d) {
RefPtr<Wrapper> wrapper =
new DemuxerProxy::Wrapper(d, taskQueue);
wrapper->UpdateBuffered();
data->mAudioDemuxer = wrapper;
DecoderDoctorLogger::LinkParentAndChild(
data->mDemuxer.get(), "decoder factory wrapper",
"MediaFormatReader::DecoderFactory::Wrapper",
wrapper.get());
}
}
data->mNumVideoTrack =
data->mDemuxer->GetNumberTracks(TrackInfo::kVideoTrack);
if (data->mNumVideoTrack) {
RefPtr<MediaTrackDemuxer> d =
data->mDemuxer->GetTrackDemuxer(TrackInfo::kVideoTrack, 0);
if (d) {
RefPtr<Wrapper> wrapper =
new DemuxerProxy::Wrapper(d, taskQueue);
wrapper->UpdateBuffered();
data->mVideoDemuxer = wrapper;
DecoderDoctorLogger::LinkParentAndChild(
data->mDemuxer.get(), "decoder factory wrapper",
"MediaFormatReader::DecoderFactory::Wrapper",
wrapper.get());
}
}
data->mCrypto = data->mDemuxer->GetCrypto();
data->mSeekable = data->mDemuxer->IsSeekable();
data->mSeekableOnlyInBufferedRange =
data->mDemuxer->IsSeekableOnlyInBufferedRanges();
data->mShouldComputeStartTime =
data->mDemuxer->ShouldComputeStartTime();
data->mInitDone = true;
return InitPromise::CreateAndResolve(NS_OK, __func__);
},
[](const MediaResult& aError) {
return InitPromise::CreateAndReject(aError, __func__);
});
}
RefPtr<MediaFormatReader::NotifyDataArrivedPromise>
MediaFormatReader::DemuxerProxy::NotifyDataArrived() {
RefPtr<Data> data = mData;
return InvokeAsync(mTaskQueue, __func__, [data]() {
if (!data->mDemuxer) {
// Was shutdown.
return NotifyDataArrivedPromise::CreateAndReject(
NS_ERROR_DOM_MEDIA_CANCELED, __func__);
}
data->mDemuxer->NotifyDataArrived();
if (data->mAudioDemuxer) {
data->mAudioDemuxer->UpdateBuffered();
}
if (data->mVideoDemuxer) {
data->mVideoDemuxer->UpdateBuffered();
}
return NotifyDataArrivedPromise::CreateAndResolve(true, __func__);
});
}
MediaFormatReader::MediaFormatReader(MediaFormatReaderInit& aInit,
MediaDataDemuxer* aDemuxer)
: mTaskQueue(new TaskQueue(GetMediaThreadPool(MediaThreadType::CONTROLLER),
"MediaFormatReader::mTaskQueue",
/* aSupportsTailDispatch = */ true)),
mAudio(this, MediaData::Type::AUDIO_DATA,
StaticPrefs::media_audio_max_decode_error()),
mVideo(this, MediaData::Type::VIDEO_DATA,
StaticPrefs::media_video_max_decode_error()),
mDemuxer(new DemuxerProxy(aDemuxer)),
mDemuxerInitDone(false),
mPendingNotifyDataArrived(false),
mLastReportedNumDecodedFrames(0),
mPreviousDecodedKeyframeTime_us(sNoPreviousDecodedKeyframe),
mKnowsCompositor(aInit.mKnowsCompositor),
mInitDone(false),
mTrackDemuxersMayBlock(false),
mSeekScheduled(false),
mVideoFrameContainer(aInit.mVideoFrameContainer),
mCrashHelper(aInit.mCrashHelper),
mDecoderFactory(new DecoderFactory(this)),
mShutdownPromisePool(new ShutdownPromisePool()),
mBuffered(mTaskQueue, TimeIntervals(),
"MediaFormatReader::mBuffered (Canonical)"),
mFrameStats(aInit.mFrameStats),
mMediaDecoderOwnerID(aInit.mMediaDecoderOwnerID) {
MOZ_ASSERT(aDemuxer);
MOZ_COUNT_CTOR(MediaFormatReader);
DDLINKCHILD("audio decoder data", "MediaFormatReader::DecoderDataWithPromise",
&mAudio);
DDLINKCHILD("video decoder data", "MediaFormatReader::DecoderDataWithPromise",
&mVideo);
DDLINKCHILD("demuxer", aDemuxer);
mOnTrackWaitingForKeyListener = OnTrackWaitingForKey().Connect(
mTaskQueue, this, &MediaFormatReader::NotifyWaitingForKey);
}
MediaFormatReader::~MediaFormatReader() {
MOZ_COUNT_DTOR(MediaFormatReader);
MOZ_ASSERT(mShutdown);
}
RefPtr<ShutdownPromise> MediaFormatReader::Shutdown() {
MOZ_ASSERT(OnTaskQueue());
LOG("");
mDemuxerInitRequest.DisconnectIfExists();
mNotifyDataArrivedPromise.DisconnectIfExists();
mMetadataPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
mSeekPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
mSkipRequest.DisconnectIfExists();
mSetCDMPromise.RejectIfExists(
MediaResult(NS_ERROR_DOM_INVALID_STATE_ERR,
"MediaFormatReader is shutting down"),
__func__);
if (mAudio.HasPromise()) {
mAudio.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
}
if (mVideo.HasPromise()) {
mVideo.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
}
if (HasAudio()) {
mAudio.ResetDemuxer();
mAudio.mTrackDemuxer->BreakCycles();
{
MutexAutoLock lock(mAudio.mMutex);
mAudio.mTrackDemuxer = nullptr;
}
mAudio.ResetState();
ShutdownDecoder(TrackInfo::kAudioTrack);
}
if (HasVideo()) {
mVideo.ResetDemuxer();
mVideo.mTrackDemuxer->BreakCycles();
{
MutexAutoLock lock(mVideo.mMutex);
mVideo.mTrackDemuxer = nullptr;
}
mVideo.ResetState();
ShutdownDecoder(TrackInfo::kVideoTrack);
}
mShutdownPromisePool->Track(mDemuxer->Shutdown());
mDemuxer = nullptr;
mOnTrackWaitingForKeyListener.Disconnect();
mShutdown = true;
return mShutdownPromisePool->Shutdown()->Then(
OwnerThread(), __func__, this, &MediaFormatReader::TearDownDecoders,
&MediaFormatReader::TearDownDecoders);
}
void MediaFormatReader::ShutdownDecoder(TrackType aTrack) {
LOGV("%s", TrackTypeToStr(aTrack));
// Shut down the pending decoder if any.
mDecoderFactory->ShutdownDecoder(aTrack);
auto& decoder = GetDecoderData(aTrack);
// Flush the decoder if necessary.
decoder.Flush();
// Shut down the decoder if any.
decoder.ShutdownDecoder();
}
void MediaFormatReader::NotifyDecoderBenchmarkStore() {
MOZ_ASSERT(OnTaskQueue());
if (!StaticPrefs::media_mediacapabilities_from_database()) {
return;
}
auto& decoder = GetDecoderData(TrackInfo::kVideoTrack);
if (decoder.GetCurrentInfo() && decoder.GetCurrentInfo()->GetAsVideoInfo()) {
VideoInfo info = *(decoder.GetCurrentInfo()->GetAsVideoInfo());
info.SetFrameRate(static_cast<int32_t>(ceil(decoder.mMeanRate.Mean())));
mOnStoreDecoderBenchmark.Notify(std::move(info));
}
}
RefPtr<ShutdownPromise> MediaFormatReader::TearDownDecoders() {
if (mAudio.mTaskQueue) {
mAudio.mTaskQueue->BeginShutdown();
mAudio.mTaskQueue->AwaitShutdownAndIdle();
mAudio.mTaskQueue = nullptr;
}
if (mVideo.mTaskQueue) {
mVideo.mTaskQueue->BeginShutdown();
mVideo.mTaskQueue->AwaitShutdownAndIdle();
mVideo.mTaskQueue = nullptr;
}
mDecoderFactory = nullptr;
mPlatform = nullptr;
mEncryptedPlatform = nullptr;
mVideoFrameContainer = nullptr;
ReleaseResources();
mBuffered.DisconnectAll();
return mTaskQueue->BeginShutdown();
}
nsresult MediaFormatReader::Init() {
MOZ_ASSERT(NS_IsMainThread(), "Must be on main thread.");
mAudio.mTaskQueue =
new TaskQueue(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER),
"MFR::mAudio::mTaskQueue");
mVideo.mTaskQueue =
new TaskQueue(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER),
"MFR::mVideo::mTaskQueue");
return NS_OK;
}
bool MediaFormatReader::ResolveSetCDMPromiseIfDone(TrackType aTrack) {
// When a CDM proxy is set, MFR would shutdown the existing MediaDataDecoder
// and would create new one for specific track in the next Update.
MOZ_ASSERT(OnTaskQueue());
if (mSetCDMPromise.IsEmpty()) {
return true;
}
MOZ_ASSERT(mCDMProxy);
if (mSetCDMForTracks.contains(aTrack)) {
mSetCDMForTracks -= aTrack;
}
if (mSetCDMForTracks.isEmpty()) {
LOGV("%s : Done ", __func__);
mSetCDMPromise.Resolve(/* aIgnored = */ true, __func__);
if (HasAudio()) {
ScheduleUpdate(TrackInfo::kAudioTrack);
}
if (HasVideo()) {
ScheduleUpdate(TrackInfo::kVideoTrack);
}
return true;
}
LOGV("%s : %s track is ready.", __func__, TrackTypeToStr(aTrack));
return false;
}
void MediaFormatReader::PrepareToSetCDMForTrack(TrackType aTrack) {
MOZ_ASSERT(OnTaskQueue());
LOGV("%s : %s", __func__, TrackTypeToStr(aTrack));
mSetCDMForTracks += aTrack;
if (mCDMProxy) {
// An old cdm proxy exists, so detaching old cdm proxy by shutting down
// MediaDataDecoder.
ShutdownDecoder(aTrack);
}
ScheduleUpdate(aTrack);
}
bool MediaFormatReader::IsDecoderWaitingForCDM(TrackType aTrack) {
MOZ_ASSERT(OnTaskQueue());
return GetDecoderData(aTrack).IsEncrypted() &&
mSetCDMForTracks.contains(aTrack) && !mCDMProxy;
}
RefPtr<SetCDMPromise> MediaFormatReader::SetCDMProxy(CDMProxy* aProxy) {
MOZ_ASSERT(OnTaskQueue());
LOGV("SetCDMProxy (%p)", aProxy);
if (mShutdown) {
return SetCDMPromise::CreateAndReject(
MediaResult(NS_ERROR_DOM_INVALID_STATE_ERR,
"MediaFormatReader is shutting down"),
__func__);
}
mSetCDMPromise.RejectIfExists(
MediaResult(NS_ERROR_DOM_INVALID_STATE_ERR,
"Another new CDM proxy is being set."),
__func__);
// Shutdown all decoders as switching CDM proxy indicates that it's
// inappropriate for the existing decoders to continue decoding via the old
// CDM proxy.
if (HasAudio()) {
PrepareToSetCDMForTrack(TrackInfo::kAudioTrack);
}
if (HasVideo()) {
PrepareToSetCDMForTrack(TrackInfo::kVideoTrack);
}
mCDMProxy = aProxy;
// Release old PDMFactory which contains an EMEDecoderModule.
mEncryptedPlatform = nullptr;
if (!mInitDone || mSetCDMForTracks.isEmpty() || !mCDMProxy) {
// 1) MFR is not initialized yet or
// 2) Demuxer is initialized without active audio and video or
// 3) A null cdm proxy is set
// the promise can be resolved directly.
mSetCDMForTracks.clear();
return SetCDMPromise::CreateAndResolve(/* aIgnored = */ true, __func__);
}
RefPtr<SetCDMPromise> p = mSetCDMPromise.Ensure(__func__);
return p;
}
bool MediaFormatReader::IsWaitingOnCDMResource() {
MOZ_ASSERT(OnTaskQueue());
return IsEncrypted() && !mCDMProxy;
}
RefPtr<MediaFormatReader::MetadataPromise>
MediaFormatReader::AsyncReadMetadata() {
AUTO_PROFILER_LABEL("MediaFormatReader::AsyncReadMetadata", MEDIA_PLAYBACK);
MOZ_ASSERT(OnTaskQueue());
MOZ_DIAGNOSTIC_ASSERT(mMetadataPromise.IsEmpty());
if (mInitDone) {
// We are returning from dormant.
MetadataHolder metadata;
metadata.mInfo = MakeUnique<MediaInfo>(mInfo);
return MetadataPromise::CreateAndResolve(std::move(metadata), __func__);
}
RefPtr<MetadataPromise> p = mMetadataPromise.Ensure(__func__);
mDemuxer->Init()
->Then(OwnerThread(), __func__, this,
&MediaFormatReader::OnDemuxerInitDone,
&MediaFormatReader::OnDemuxerInitFailed)
->Track(mDemuxerInitRequest);
return p;
}
void MediaFormatReader::OnDemuxerInitDone(const MediaResult& aResult) {
AUTO_PROFILER_LABEL("MediaFormatReader::OnDemuxerInitDone", MEDIA_PLAYBACK);
MOZ_ASSERT(OnTaskQueue());
mDemuxerInitRequest.Complete();
if (NS_FAILED(aResult) && StaticPrefs::media_playback_warnings_as_errors()) {
mMetadataPromise.Reject(aResult, __func__);
return;
}
mDemuxerInitDone = true;
UniquePtr<MetadataTags> tags(MakeUnique<MetadataTags>());
RefPtr<PDMFactory> platform;
if (!IsWaitingOnCDMResource()) {
platform = new PDMFactory();
}
// To decode, we need valid video and a place to put it.
bool videoActive = !!mDemuxer->GetNumberTracks(TrackInfo::kVideoTrack) &&
GetImageContainer();
if (videoActive) {
// We currently only handle the first video track.
MutexAutoLock lock(mVideo.mMutex);
mVideo.mTrackDemuxer = mDemuxer->GetTrackDemuxer(TrackInfo::kVideoTrack, 0);
if (!mVideo.mTrackDemuxer) {
mMetadataPromise.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR, __func__);
return;
}
UniquePtr<TrackInfo> videoInfo = mVideo.mTrackDemuxer->GetInfo();
videoActive = videoInfo && videoInfo->IsValid();
if (videoActive) {
if (platform &&
!platform->SupportsMimeType(videoInfo->mMimeType, nullptr)) {
// We have no decoder for this track. Error.
mMetadataPromise.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR, __func__);
return;
}
mInfo.mVideo = *videoInfo->GetAsVideoInfo();
mVideo.mWorkingInfo = MakeUnique<VideoInfo>(mInfo.mVideo);
for (const MetadataTag& tag : videoInfo->mTags) {
tags->Put(tag.mKey, tag.mValue);
}
mVideo.mOriginalInfo = std::move(videoInfo);
mTrackDemuxersMayBlock |= mVideo.mTrackDemuxer->GetSamplesMayBlock();
} else {
mVideo.mTrackDemuxer->BreakCycles();
mVideo.mTrackDemuxer = nullptr;
}
}
bool audioActive = !!mDemuxer->GetNumberTracks(TrackInfo::kAudioTrack);
if (audioActive) {
MutexAutoLock lock(mAudio.mMutex);
mAudio.mTrackDemuxer = mDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0);
if (!mAudio.mTrackDemuxer) {
mMetadataPromise.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR, __func__);
return;
}
UniquePtr<TrackInfo> audioInfo = mAudio.mTrackDemuxer->GetInfo();
// We actively ignore audio tracks that we know we can't play.
audioActive = audioInfo && audioInfo->IsValid() &&
(!platform ||
platform->SupportsMimeType(audioInfo->mMimeType, nullptr));
if (audioActive) {
mInfo.mAudio = *audioInfo->GetAsAudioInfo();
mAudio.mWorkingInfo = MakeUnique<AudioInfo>(mInfo.mAudio);
for (const MetadataTag& tag : audioInfo->mTags) {
tags->Put(tag.mKey, tag.mValue);
}
mAudio.mOriginalInfo = std::move(audioInfo);
mTrackDemuxersMayBlock |= mAudio.mTrackDemuxer->GetSamplesMayBlock();
} else {
mAudio.mTrackDemuxer->BreakCycles();
mAudio.mTrackDemuxer = nullptr;
}
}
UniquePtr<EncryptionInfo> crypto = mDemuxer->GetCrypto();
if (crypto && crypto->IsEncrypted()) {
// Try and dispatch 'encrypted'. Won't go if ready state still HAVE_NOTHING.
for (uint32_t i = 0; i < crypto->mInitDatas.Length(); i++) {
mOnEncrypted.Notify(crypto->mInitDatas[i].mInitData,
crypto->mInitDatas[i].mType);
}
mInfo.mCrypto = *crypto;
}
auto videoDuration = HasVideo() ? mInfo.mVideo.mDuration : TimeUnit::Zero();
auto audioDuration = HasAudio() ? mInfo.mAudio.mDuration : TimeUnit::Zero();
auto duration = std::max(videoDuration, audioDuration);
if (duration.IsPositive()) {
mInfo.mMetadataDuration = Some(duration);
}
mInfo.mMediaSeekable = mDemuxer->IsSeekable();
mInfo.mMediaSeekableOnlyInBufferedRanges =
mDemuxer->IsSeekableOnlyInBufferedRanges();
if (!videoActive && !audioActive) {
mMetadataPromise.Reject(NS_ERROR_DOM_MEDIA_METADATA_ERR, __func__);
return;
}
mTags = std::move(tags);
mInitDone = true;
// Try to get the start time.
// For MSE case, the start time of each track is assumed to be 0.
// For others, we must demux the first sample to know the start time for each
// track.
if (!mDemuxer->ShouldComputeStartTime()) {
mAudio.mFirstDemuxedSampleTime.emplace(TimeUnit::Zero());
mVideo.mFirstDemuxedSampleTime.emplace(TimeUnit::Zero());
} else {
if (HasAudio()) {
RequestDemuxSamples(TrackInfo::kAudioTrack);
}
if (HasVideo()) {
RequestDemuxSamples(TrackInfo::kVideoTrack);
}
}
if (aResult != NS_OK) {
mOnDecodeWarning.Notify(aResult);
}
MaybeResolveMetadataPromise();
}
void MediaFormatReader::MaybeResolveMetadataPromise() {
MOZ_ASSERT(OnTaskQueue());
if ((HasAudio() && mAudio.mFirstDemuxedSampleTime.isNothing()) ||
(HasVideo() && mVideo.mFirstDemuxedSampleTime.isNothing())) {
return;
}
TimeUnit startTime =
std::min(mAudio.mFirstDemuxedSampleTime.refOr(TimeUnit::FromInfinity()),
mVideo.mFirstDemuxedSampleTime.refOr(TimeUnit::FromInfinity()));
if (!startTime.IsInfinite()) {
mInfo.mStartTime = startTime; // mInfo.mStartTime is initialized to 0.
}
MetadataHolder metadata;
metadata.mInfo = MakeUnique<MediaInfo>(mInfo);
metadata.mTags = mTags->Count() ? std::move(mTags) : nullptr;
// We now have all the informations required to calculate the initial buffered
// range.
mHasStartTime = true;
UpdateBuffered();
mMetadataPromise.Resolve(std::move(metadata), __func__);
}
bool MediaFormatReader::IsEncrypted() const {
return (HasAudio() && mAudio.GetCurrentInfo()->mCrypto.IsEncrypted()) ||
(HasVideo() && mVideo.GetCurrentInfo()->mCrypto.IsEncrypted());
}
void MediaFormatReader::OnDemuxerInitFailed(const MediaResult& aError) {
mDemuxerInitRequest.Complete();
mMetadataPromise.Reject(aError, __func__);
}
void MediaFormatReader::ReadUpdatedMetadata(MediaInfo* aInfo) {
// Called on the MDSM's TaskQueue.
{
MutexAutoLock lock(mVideo.mMutex);
if (HasVideo()) {
aInfo->mVideo = *mVideo.GetWorkingInfo()->GetAsVideoInfo();
}
}
{
MutexAutoLock lock(mAudio.mMutex);
if (HasAudio()) {
aInfo->mAudio = *mAudio.GetWorkingInfo()->GetAsAudioInfo();
}
}
}
MediaFormatReader::DecoderData& MediaFormatReader::GetDecoderData(
TrackType aTrack) {
MOZ_ASSERT(aTrack == TrackInfo::kAudioTrack ||
aTrack == TrackInfo::kVideoTrack);
if (aTrack == TrackInfo::kAudioTrack) {
return mAudio;
}
return mVideo;
}
bool MediaFormatReader::ShouldSkip(TimeUnit aTimeThreshold) {
MOZ_ASSERT(HasVideo());
if (!StaticPrefs::media_decoder_skip_to_next_key_frame_enabled()) {
return false;
}
TimeUnit nextKeyframe;
nsresult rv = mVideo.mTrackDemuxer->GetNextRandomAccessPoint(&nextKeyframe);
if (NS_FAILED(rv)) {
// Only OggTrackDemuxer with video type gets into here.
// We don't support skip-to-next-frame for this case.
return false;
}
return (nextKeyframe <= aTimeThreshold ||
(mVideo.mTimeThreshold &&
mVideo.mTimeThreshold.ref().EndTime() < aTimeThreshold)) &&
nextKeyframe.ToMicroseconds() >= 0 && !nextKeyframe.IsInfinite();
}
RefPtr<MediaFormatReader::VideoDataPromise> MediaFormatReader::RequestVideoData(
const TimeUnit& aTimeThreshold) {
MOZ_ASSERT(OnTaskQueue());
MOZ_DIAGNOSTIC_ASSERT(mSeekPromise.IsEmpty(),
"No sample requests allowed while seeking");
MOZ_DIAGNOSTIC_ASSERT(!mVideo.HasPromise(), "No duplicate sample requests");
MOZ_DIAGNOSTIC_ASSERT(!mVideo.mSeekRequest.Exists() ||
mVideo.mTimeThreshold.isSome());
MOZ_DIAGNOSTIC_ASSERT(!IsSeeking(), "called mid-seek");
LOGV("RequestVideoData(%" PRId64 ")", aTimeThreshold.ToMicroseconds());
if (!HasVideo()) {
LOG("called with no video track");
return VideoDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR,
__func__);
}
if (IsSeeking()) {
LOG("called mid-seek. Rejecting.");
return VideoDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED,
__func__);
}
if (mShutdown) {
NS_WARNING("RequestVideoData on shutdown MediaFormatReader!");
return VideoDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED,
__func__);
}
// Ensure we have no pending seek going as ShouldSkip could return out of date
// information.
if (!mVideo.HasInternalSeekPending() && ShouldSkip(aTimeThreshold)) {
RefPtr<VideoDataPromise> p = mVideo.EnsurePromise(__func__);
SkipVideoDemuxToNextKeyFrame(aTimeThreshold);
return p;
}
RefPtr<VideoDataPromise> p = mVideo.EnsurePromise(__func__);
ScheduleUpdate(TrackInfo::kVideoTrack);
return p;
}
void MediaFormatReader::OnDemuxFailed(TrackType aTrack,
const MediaResult& aError) {
AUTO_PROFILER_LABEL("MediaFormatReader::OnDemuxFailed", MEDIA_PLAYBACK);
MOZ_ASSERT(OnTaskQueue());
LOG("Failed to demux %s, failure:%s",
aTrack == TrackType::kVideoTrack ? "video" : "audio",
aError.ErrorName().get());
auto& decoder = GetDecoderData(aTrack);
decoder.mDemuxRequest.Complete();
switch (aError.Code()) {
case NS_ERROR_DOM_MEDIA_END_OF_STREAM:
DDLOG(DDLogCategory::Log,
aTrack == TrackType::kVideoTrack ? "video_demux_interruption"
: "audio_demux_interruption",
aError);
if (!decoder.mWaitingForData) {
decoder.RequestDrain();
}
NotifyEndOfStream(aTrack);
break;
case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA:
DDLOG(DDLogCategory::Log,
aTrack == TrackType::kVideoTrack ? "video_demux_interruption"
: "audio_demux_interruption",
aError);
if (!decoder.mWaitingForData) {
decoder.RequestDrain();
}
NotifyWaitingForData(aTrack);
break;
case NS_ERROR_DOM_MEDIA_CANCELED:
DDLOG(DDLogCategory::Log,
aTrack == TrackType::kVideoTrack ? "video_demux_interruption"
: "audio_demux_interruption",
aError);
if (decoder.HasPromise()) {
decoder.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
}
break;
default:
DDLOG(DDLogCategory::Log,
aTrack == TrackType::kVideoTrack ? "video_demux_error"
: "audio_demux_error",
aError);
NotifyError(aTrack, aError);
break;
}
}
void MediaFormatReader::DoDemuxVideo() {
AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxVideo", MEDIA_PLAYBACK);
using SamplesPromise = MediaTrackDemuxer::SamplesPromise;
DDLOG(DDLogCategory::Log, "video_demuxing", DDNoValue{});
auto p = mVideo.mTrackDemuxer->GetSamples(1);
if (mVideo.mFirstDemuxedSampleTime.isNothing()) {
RefPtr<MediaFormatReader> self = this;
p = p->Then(
OwnerThread(), __func__,
[self](RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxVideo:Resolved",
MEDIA_PLAYBACK);
DDLOGEX(self.get(), DDLogCategory::Log, "video_first_demuxed",
DDNoValue{});
self->OnFirstDemuxCompleted(TrackInfo::kVideoTrack, aSamples);
return SamplesPromise::CreateAndResolve(aSamples.forget(), __func__);
},
[self](const MediaResult& aError) {
AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxVideo:Rejected",
MEDIA_PLAYBACK);
DDLOGEX(self.get(), DDLogCategory::Log, "video_first_demuxing_error",
aError);
self->OnFirstDemuxFailed(TrackInfo::kVideoTrack, aError);
return SamplesPromise::CreateAndReject(aError, __func__);
});
}
p->Then(OwnerThread(), __func__, this,
&MediaFormatReader::OnVideoDemuxCompleted,
&MediaFormatReader::OnVideoDemuxFailed)
->Track(mVideo.mDemuxRequest);
}
void MediaFormatReader::OnVideoDemuxCompleted(
RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
AUTO_PROFILER_LABEL("MediaFormatReader::OnVideoDemuxCompleted",
MEDIA_PLAYBACK);
LOGV("%zu video samples demuxed (sid:%d)", aSamples->GetSamples().Length(),
aSamples->GetSamples()[0]->mTrackInfo
? aSamples->GetSamples()[0]->mTrackInfo->GetID()
: 0);
DDLOG(DDLogCategory::Log, "video_demuxed_samples",
uint64_t(aSamples->GetSamples().Length()));
mVideo.mDemuxRequest.Complete();
mVideo.mQueuedSamples.AppendElements(aSamples->GetSamples());
ScheduleUpdate(TrackInfo::kVideoTrack);
}
RefPtr<MediaFormatReader::AudioDataPromise>
MediaFormatReader::RequestAudioData() {
MOZ_ASSERT(OnTaskQueue());
MOZ_DIAGNOSTIC_ASSERT(!mAudio.HasPromise(), "No duplicate sample requests");
MOZ_DIAGNOSTIC_ASSERT(IsVideoSeeking() || mSeekPromise.IsEmpty(),
"No sample requests allowed while seeking");
MOZ_DIAGNOSTIC_ASSERT(IsVideoSeeking() || !mAudio.mSeekRequest.Exists() ||
mAudio.mTimeThreshold.isSome());
MOZ_DIAGNOSTIC_ASSERT(IsVideoSeeking() || !IsSeeking(), "called mid-seek");
LOGV("");
if (!HasAudio()) {
LOG("called with no audio track");
return AudioDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_FATAL_ERR,
__func__);
}
if (IsSeeking()) {
LOG("called mid-seek. Rejecting.");
return AudioDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED,
__func__);
}
if (mShutdown) {
NS_WARNING("RequestAudioData on shutdown MediaFormatReader!");
return AudioDataPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED,
__func__);
}
RefPtr<AudioDataPromise> p = mAudio.EnsurePromise(__func__);
ScheduleUpdate(TrackInfo::kAudioTrack);
return p;
}
void MediaFormatReader::DoDemuxAudio() {
AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxAudio", MEDIA_PLAYBACK);
using SamplesPromise = MediaTrackDemuxer::SamplesPromise;
DDLOG(DDLogCategory::Log, "audio_demuxing", DDNoValue{});
auto p = mAudio.mTrackDemuxer->GetSamples(1);
if (mAudio.mFirstDemuxedSampleTime.isNothing()) {
RefPtr<MediaFormatReader> self = this;
p = p->Then(
OwnerThread(), __func__,
[self](RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxAudio:Resolved",
MEDIA_PLAYBACK);
DDLOGEX(self.get(), DDLogCategory::Log, "audio_first_demuxed",
DDNoValue{});
self->OnFirstDemuxCompleted(TrackInfo::kAudioTrack, aSamples);
return SamplesPromise::CreateAndResolve(aSamples.forget(), __func__);
},
[self](const MediaResult& aError) {
AUTO_PROFILER_LABEL("MediaFormatReader::DoDemuxAudio:Rejected",
MEDIA_PLAYBACK);
DDLOGEX(self.get(), DDLogCategory::Log, "audio_first_demuxing_error",
aError);
self->OnFirstDemuxFailed(TrackInfo::kAudioTrack, aError);
return SamplesPromise::CreateAndReject(aError, __func__);
});
}
p->Then(OwnerThread(), __func__, this,
&MediaFormatReader::OnAudioDemuxCompleted,
&MediaFormatReader::OnAudioDemuxFailed)
->Track(mAudio.mDemuxRequest);
}
void MediaFormatReader::OnAudioDemuxCompleted(
RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
LOGV("%zu audio samples demuxed (sid:%d)", aSamples->GetSamples().Length(),
aSamples->GetSamples()[0]->mTrackInfo
? aSamples->GetSamples()[0]->mTrackInfo->GetID()
: 0);
DDLOG(DDLogCategory::Log, "audio_demuxed_samples",
uint64_t(aSamples->GetSamples().Length()));
mAudio.mDemuxRequest.Complete();
mAudio.mQueuedSamples.AppendElements(aSamples->GetSamples());
ScheduleUpdate(TrackInfo::kAudioTrack);
}
void MediaFormatReader::NotifyNewOutput(
TrackType aTrack, MediaDataDecoder::DecodedData&& aResults) {
AUTO_PROFILER_LABEL("MediaFormatReader::NotifyNewOutput", MEDIA_PLAYBACK);
MOZ_ASSERT(OnTaskQueue());
auto& decoder = GetDecoderData(aTrack);
if (aResults.IsEmpty()) {
DDLOG(DDLogCategory::Log,
aTrack == TrackInfo::kAudioTrack ? "decoded_audio" : "decoded_video",
"no output samples");
} else
for (auto&& sample : aResults) {
if (DecoderDoctorLogger::IsDDLoggingEnabled()) {
switch (sample->mType) {
case MediaData::Type::AUDIO_DATA:
DDLOGPR(DDLogCategory::Log,
aTrack == TrackInfo::kAudioTrack ? "decoded_audio"
: "decoded_got_audio!?",
"{\"type\":\"AudioData\", \"offset\":%" PRIi64
", \"time_us\":%" PRIi64 ", \"timecode_us\":%" PRIi64
", \"duration_us\":%" PRIi64 ", \"frames\":%" PRIu32
", \"channels\":%" PRIu32 ", \"rate\":%" PRIu32
", \"bytes\":%zu}",
sample->mOffset, sample->mTime.ToMicroseconds(),
sample->mTimecode.ToMicroseconds(),
sample->mDuration.ToMicroseconds(),
sample->As<AudioData>()->Frames(),
sample->As<AudioData>()->mChannels,
sample->As<AudioData>()->mRate,
sample->As<AudioData>()->Data().Length());
break;
case MediaData::Type::VIDEO_DATA:
DDLOGPR(DDLogCategory::Log,
aTrack == TrackInfo::kVideoTrack ? "decoded_video"
: "decoded_got_video!?",
"{\"type\":\"VideoData\", \"offset\":%" PRIi64
", \"time_us\":%" PRIi64 ", \"timecode_us\":%" PRIi64
", \"duration_us\":%" PRIi64
", \"kf\":%s, \"size\":[%" PRIi32 ",%" PRIi32 "]}",
sample->mOffset, sample->mTime.ToMicroseconds(),
sample->mTimecode.ToMicroseconds(),
sample->mDuration.ToMicroseconds(),
sample->mKeyframe ? "true" : "false",
sample->As<VideoData>()->mDisplay.width,
sample->As<VideoData>()->mDisplay.height);
break;
case MediaData::Type::RAW_DATA:
DDLOGPR(DDLogCategory::Log,
aTrack == TrackInfo::kAudioTrack
? "decoded_audio"
: aTrack == TrackInfo::kVideoTrack ? "decoded_video"
: "decoded_?",
"{\"type\":\"RawData\", \"offset\":%" PRIi64
" \"time_us\":%" PRIi64 ", \"timecode_us\":%" PRIi64
", \"duration_us\":%" PRIi64 ", \"kf\":%s}",
sample->mOffset, sample->mTime.ToMicroseconds(),
sample->mTimecode.ToMicroseconds(),
sample->mDuration.ToMicroseconds(),
sample->mKeyframe ? "true" : "false");
break;
case MediaData::Type::NULL_DATA:
DDLOGPR(DDLogCategory::Log,
aTrack == TrackInfo::kAudioTrack
? "decoded_audio"
: aTrack == TrackInfo::kVideoTrack ? "decoded_video"
: "decoded_?",
"{\"type\":\"NullData\", \"offset\":%" PRIi64
" \"time_us\":%" PRIi64 ", \"timecode_us\":%" PRIi64
", \"duration_us\":%" PRIi64 ", \"kf\":%s}",
sample->mOffset, sample->mTime.ToMicroseconds(),
sample->mTimecode.ToMicroseconds(),
sample->mDuration.ToMicroseconds(),
sample->mKeyframe ? "true" : "false");
break;
}
}
LOGV("Received new %s sample time:%" PRId64 " duration:%" PRId64,
TrackTypeToStr(aTrack), sample->mTime.ToMicroseconds(),
sample->mDuration.ToMicroseconds());
decoder.mOutput.AppendElement(sample);
decoder.mNumSamplesOutput++;
decoder.mNumOfConsecutiveError = 0;
}
LOG("Done processing new %s samples", TrackTypeToStr(aTrack));
if (!aResults.IsEmpty()) {
// We have decoded our first frame, we can now starts to skip future errors.
decoder.mFirstFrameTime.reset();
}
ScheduleUpdate(aTrack);
}
void MediaFormatReader::NotifyError(TrackType aTrack,
const MediaResult& aError) {
MOZ_ASSERT(OnTaskQueue());
NS_WARNING(aError.Description().get());
LOGV("%s Decoding error", TrackTypeToStr(aTrack));
auto& decoder = GetDecoderData(aTrack);
decoder.mError = decoder.HasFatalError() ? decoder.mError : Some(aError);
ScheduleUpdate(aTrack);
}
void MediaFormatReader::NotifyWaitingForData(TrackType aTrack) {
MOZ_ASSERT(OnTaskQueue());
auto& decoder = GetDecoderData(aTrack);
decoder.mWaitingForData = true;
if (decoder.mTimeThreshold) {
decoder.mTimeThreshold.ref().mWaiting = true;
}
ScheduleUpdate(aTrack);
}
void MediaFormatReader::NotifyWaitingForKey(TrackType aTrack) {
MOZ_ASSERT(OnTaskQueue());
auto& decoder = GetDecoderData(aTrack);
mOnWaitingForKey.Notify();
if (!decoder.mDecodeRequest.Exists()) {
LOGV("WaitingForKey received while no pending decode. Ignoring");
return;
}
decoder.mWaitingForKey = true;
ScheduleUpdate(aTrack);
}
void MediaFormatReader::NotifyEndOfStream(TrackType aTrack) {
MOZ_ASSERT(OnTaskQueue());
auto& decoder = GetDecoderData(aTrack);
decoder.mDemuxEOS = true;
ScheduleUpdate(aTrack);
}
bool MediaFormatReader::NeedInput(DecoderData& aDecoder) {
// The decoder will not be fed a new raw sample until the current decoding
// requests has completed.
return (aDecoder.HasPromise() || aDecoder.mTimeThreshold.isSome()) &&
!aDecoder.HasPendingDrain() && !aDecoder.HasFatalError() &&
!aDecoder.mDemuxRequest.Exists() && !aDecoder.mOutput.Length() &&
!aDecoder.HasInternalSeekPending() &&
!aDecoder.mDecodeRequest.Exists();
}
void MediaFormatReader::ScheduleUpdate(TrackType aTrack) {
MOZ_ASSERT(OnTaskQueue());
if (mShutdown) {
return;
}
auto& decoder = GetDecoderData(aTrack);
MOZ_RELEASE_ASSERT(decoder.GetCurrentInfo(),
"Can only schedule update when track exists");
if (decoder.mUpdateScheduled) {
return;
}
LOGV("SchedulingUpdate(%s)", TrackTypeToStr(aTrack));
decoder.mUpdateScheduled = true;
RefPtr<nsIRunnable> task(NewRunnableMethod<TrackType>(
"MediaFormatReader::Update", this, &MediaFormatReader::Update, aTrack));
nsresult rv = OwnerThread()->Dispatch(task.forget());
MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
Unused << rv;
}
bool MediaFormatReader::UpdateReceivedNewData(TrackType aTrack) {
MOZ_ASSERT(OnTaskQueue());
auto& decoder = GetDecoderData(aTrack);
if (!decoder.mReceivedNewData) {
return false;
}
// We do not want to clear mWaitingForData while there are pending
// demuxing or seeking operations that could affect the value of this flag.
// This is in order to ensure that we will retry once they complete as we may
// now have new data that could potentially allow those operations to
// successfully complete if tried again.
if (decoder.mSeekRequest.Exists()) {
// Nothing more to do until this operation complete.
return true;
}
if (aTrack == TrackType::kVideoTrack && mSkipRequest.Exists()) {
LOGV("Skipping in progress, nothing more to do");
return true;
}
if (decoder.mDemuxRequest.Exists()) {
// We may have pending operations to process, so we want to continue
// after UpdateReceivedNewData returns.
return false;
}
if (decoder.HasPendingDrain()) {
// We do not want to clear mWaitingForData or mDemuxEOS while
// a drain is in progress in order to properly complete the operation.
return false;
}
decoder.mReceivedNewData = false;
if (decoder.mTimeThreshold) {
decoder.mTimeThreshold.ref().mWaiting = false;
}
decoder.mWaitingForData = false;
if (decoder.HasFatalError()) {
return false;
}
if (!mSeekPromise.IsEmpty() &&
(!IsVideoSeeking() || aTrack == TrackInfo::kVideoTrack)) {
MOZ_ASSERT(!decoder.HasPromise());
MOZ_DIAGNOSTIC_ASSERT(
(IsVideoSeeking() || !