MediaFormatReader.cpp

mozilla-central/dom/media/MediaFormatReader.cpp (file symbol)

Enable keyboard shortcuts

Source code

Revision control

Copy as Markdown

Other Tools

/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */

/* vim:set ts=2 sw=2 sts=2 et cindent: */

/* This Source Code Form is subject to the terms of the Mozilla Public

 * License, v. 2.0. If a copy of the MPL was not distributed with this

 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#include "MediaFormatReader.h"

#include <algorithm>

#include <map>

#include <queue>

#include "AllocationPolicy.h"

#ifdef MOZ_AV1

#  include "AOMDecoder.h"

#endif

#include "DecoderBenchmark.h"

#include "MediaData.h"

#include "MediaDataDecoderProxy.h"

#include "MediaInfo.h"

#include "MP4Decoder.h"

#include "PDMFactory.h"

#include "PerformanceRecorder.h"

#include "VideoFrameContainer.h"

#include "VideoUtils.h"

#include "VPXDecoder.h"

#include "mozilla/AbstractThread.h"

#include "mozilla/CDMProxy.h"

#include "mozilla/ClearOnShutdown.h"

#include "mozilla/NotNull.h"

#include "mozilla/Preferences.h"

#include "mozilla/ProfilerLabels.h"

#include "mozilla/ProfilerMarkers.h"

#include "mozilla/SharedThreadPool.h"

#include "mozilla/StaticPrefs_media.h"

#include "mozilla/TaskQueue.h"

#include "mozilla/Unused.h"

#include "mozilla/glean/GleanMetrics.h"

#include "nsContentUtils.h"

#include "nsLiteralString.h"

#include "nsPrintfCString.h"

#include "nsTHashSet.h"

using namespace mozilla::media;

static mozilla::LazyLogModule sFormatDecoderLog("MediaFormatReader");

mozilla::LazyLogModule gMediaDemuxerLog("MediaDemuxer");

#define LOG(arg, ...)                                                  \

  DDMOZ_LOG(sFormatDecoderLog, mozilla::LogLevel::Debug, "::%s: " arg, \

            __func__, ##__VA_ARGS__)

#define LOGV(arg, ...)                                                   \

  DDMOZ_LOG(sFormatDecoderLog, mozilla::LogLevel::Verbose, "::%s: " arg, \

            __func__, ##__VA_ARGS__)

#define NS_DispatchToMainThread(...) CompileError_UseAbstractMainThreadInstead

namespace mozilla {

using MediaDataDecoderID = void*;

/**

 * This class tracks shutdown promises to ensure all decoders are shut down

 * completely before MFR continues the rest of the shutdown procedure.

*/

class MediaFormatReader::ShutdownPromisePool {

 public:

  ShutdownPromisePool()

      : mOnShutdownComplete(new ShutdownPromise::Private(__func__)) {}

  // Return a promise which will be resolved when all the tracking promises

  // are resolved. Note no more promises should be added for tracking once

  // this function is called.

  RefPtr<ShutdownPromise> Shutdown();

  // Track a shutdown promise.

  void Track(const RefPtr<ShutdownPromise>& aPromise);

  // Shut down a decoder and track its shutdown promise.

  void ShutdownDecoder(already_AddRefed<MediaDataDecoder> aDecoder) {

    Track(RefPtr<MediaDataDecoder>(aDecoder)->Shutdown());

 private:

  bool mShutdown = false;

  const RefPtr<ShutdownPromise::Private> mOnShutdownComplete;

  nsTHashSet<RefPtr<ShutdownPromise>> mPromises;

};

RefPtr<ShutdownPromise> MediaFormatReader::ShutdownPromisePool::Shutdown() {

  MOZ_DIAGNOSTIC_ASSERT(!mShutdown);

  mShutdown = true;

  if (mPromises.Count() == 0) {

    mOnShutdownComplete->Resolve(true, __func__);

  return mOnShutdownComplete;

void MediaFormatReader::ShutdownPromisePool::Track(

    const RefPtr<ShutdownPromise>& aPromise) {

  MOZ_DIAGNOSTIC_ASSERT(!mShutdown);

  MOZ_DIAGNOSTIC_ASSERT(!mPromises.Contains(aPromise));

  mPromises.Insert(aPromise);

  aPromise->Then(AbstractThread::GetCurrent(), __func__, [aPromise, this]() {

    MOZ_DIAGNOSTIC_ASSERT(mPromises.Contains(aPromise));

    mPromises.Remove(aPromise);

    if (mShutdown && mPromises.Count() == 0) {

      mOnShutdownComplete->Resolve(true, __func__);

});

void MediaFormatReader::DecoderData::ShutdownDecoder() {

  MOZ_ASSERT(mOwner->OnTaskQueue());

  MutexAutoLock lock(mMutex);

  if (!mDecoder) {

    // No decoder to shut down.

    return;

  if (mFlushing) {

    // Flush is is in action. Shutdown will be initiated after flush completes.

    MOZ_DIAGNOSTIC_ASSERT(mShutdownPromise);

    mOwner->mShutdownPromisePool->Track(mShutdownPromise->Ensure(__func__));

    // The order of decoder creation and shutdown is handled by LocalAllocPolicy

    // and ShutdownPromisePool. MFR can now reset these members to a fresh state

    // and be ready to create new decoders again without explicitly waiting for

    // flush/shutdown to complete.

    mShutdownPromise = nullptr;

    mFlushing = false;

  } else {

    // No flush is in action. We can shut down the decoder now.

    mOwner->mShutdownPromisePool->Track(mDecoder->Shutdown());

  // mShutdownPromisePool will handle the order of decoder shutdown so

  // we can forget mDecoder and be ready to create a new one.

  mDecoder = nullptr;

  mDescription = "shutdown"_ns;

  mHasReportedVideoHardwareSupportTelemtry = false;

  mOwner->ScheduleUpdate(mType == MediaData::Type::AUDIO_DATA

                             ? TrackType::kAudioTrack

                             : TrackType::kVideoTrack);

void MediaFormatReader::DecoderData::Flush() {

  AUTO_PROFILER_LABEL("MediaFormatReader::Flush", MEDIA_PLAYBACK);

  MOZ_ASSERT(mOwner->OnTaskQueue());

  if (mFlushing || mFlushed) {

    // Flush still pending or already flushed, nothing more to do.

    return;

  mDecodeRequest.DisconnectIfExists();

  mDrainRequest.DisconnectIfExists();

  mDrainState = DrainState::None;

  CancelWaitingForKey();

  mOutput.Clear();

  mNumSamplesInput = 0;

  mNumSamplesOutput = 0;

  mSizeOfQueue = 0;

  if (mDecoder) {

    TrackType type = mType == MediaData::Type::AUDIO_DATA

                         ? TrackType::kAudioTrack

                         : TrackType::kVideoTrack;

    mFlushing = true;

    MOZ_DIAGNOSTIC_ASSERT(!mShutdownPromise);

    mShutdownPromise = new SharedShutdownPromiseHolder();

    RefPtr<SharedShutdownPromiseHolder> p = mShutdownPromise;

    RefPtr<MediaDataDecoder> d = mDecoder;

    DDLOGEX2("MediaFormatReader::DecoderData", this, DDLogCategory::Log,

             "flushing", DDNoValue{});

    mDecoder->Flush()->Then(

        mOwner->OwnerThread(), __func__,

        [type, this, p, d]() {

          AUTO_PROFILER_LABEL("MediaFormatReader::Flush:Resolved",

                              MEDIA_PLAYBACK);

          DDLOGEX2("MediaFormatReader::DecoderData", this, DDLogCategory::Log,

                   "flushed", DDNoValue{});

          if (!p->IsEmpty()) {

            // Shutdown happened before flush completes.

            // Let's continue to shut down the decoder. Note

            // we don't access |this| because this decoder

            // is no longer managed by MFR::DecoderData.

            d->Shutdown()->ChainTo(p->Steal(), __func__);

            return;

          mFlushing = false;

          mShutdownPromise = nullptr;

          mOwner->ScheduleUpdate(type);

},

        [type, this, p, d](const MediaResult& aError) {

          AUTO_PROFILER_LABEL("MediaFormatReader::Flush:Rejected",

                              MEDIA_PLAYBACK);

          DDLOGEX2("MediaFormatReader::DecoderData", this, DDLogCategory::Log,

                   "flush_error", aError);

          if (!p->IsEmpty()) {

            d->Shutdown()->ChainTo(p->Steal(), __func__);

            return;

          mFlushing = false;

          mShutdownPromise = nullptr;

          mOwner->NotifyError(type, aError);

});

  mFlushed = true;

class MediaFormatReader::DecoderFactory {

  using InitPromise = MediaDataDecoder::InitPromise;

  using TokenPromise = AllocPolicy::Promise;

  using Token = AllocPolicy::Token;

  using CreateDecoderPromise = PlatformDecoderModule::CreateDecoderPromise;

 public:

  explicit DecoderFactory(MediaFormatReader* aOwner)

      : mAudio(aOwner->mAudio, TrackInfo::kAudioTrack, aOwner->OwnerThread()),

        mVideo(aOwner->mVideo, TrackInfo::kVideoTrack, aOwner->OwnerThread()),

        mOwner(WrapNotNull(aOwner)) {

    DecoderDoctorLogger::LogConstruction("MediaFormatReader::DecoderFactory",

                                         this);

    DecoderDoctorLogger::LinkParentAndChild(

        aOwner, "decoder factory", "MediaFormatReader::DecoderFactory", this);

  ~DecoderFactory() {

    DecoderDoctorLogger::LogDestruction("MediaFormatReader::DecoderFactory",

                                        this);

  void CreateDecoder(TrackType aTrack);

  // Shutdown any decoder pending initialization and reset mAudio/mVideo to its

  // pristine state so CreateDecoder() is ready to be called again immediately.

  void ShutdownDecoder(TrackType aTrack) {

    MOZ_ASSERT(aTrack == TrackInfo::kAudioTrack ||

               aTrack == TrackInfo::kVideoTrack);

    auto& data = aTrack == TrackInfo::kAudioTrack ? mAudio : mVideo;

    data.mPolicy->Cancel();

    data.mTokenRequest.DisconnectIfExists();

    if (data.mLiveToken) {

      // We haven't completed creation of the decoder, and it hasn't been

      // initialised yet.

      data.mLiveToken = nullptr;

      // The decoder will be shutdown as soon as it's available and tracked by

      // the ShutdownPromisePool.

      mOwner->mShutdownPromisePool->Track(data.mCreateDecoderPromise->Then(

          mOwner->mTaskQueue, __func__,

          [](CreateDecoderPromise::ResolveOrRejectValue&& aResult) {

            if (aResult.IsReject()) {

              return ShutdownPromise::CreateAndResolve(true, __func__);

            return aResult.ResolveValue()->Shutdown();

          }));

      // Free the token to leave room for a new decoder.

      data.mToken = nullptr;

    data.mInitRequest.DisconnectIfExists();

    if (data.mDecoder) {

      mOwner->mShutdownPromisePool->ShutdownDecoder(data.mDecoder.forget());

    data.mStage = Stage::None;

    MOZ_ASSERT(!data.mToken);

 private:

  enum class Stage : int8_t { None, WaitForToken, CreateDecoder, WaitForInit };

  struct Data {

    Data(DecoderData& aOwnerData, TrackType aTrack, TaskQueue* aThread)

        : mOwnerData(aOwnerData),

          mTrack(aTrack),

          mPolicy(new SingleAllocPolicy(aTrack, aThread)) {}

    DecoderData& mOwnerData;

    const TrackType mTrack;

    RefPtr<SingleAllocPolicy> mPolicy;

    Stage mStage = Stage::None;

    RefPtr<Token> mToken;

    RefPtr<MediaDataDecoder> mDecoder;

    MozPromiseRequestHolder<TokenPromise> mTokenRequest;

    struct DecoderCancelled : public SupportsWeakPtr {

      NS_INLINE_DECL_REFCOUNTING_ONEVENTTARGET(DecoderCancelled)

     private:

      ~DecoderCancelled() = default;

};

    // Set when decoder is about to be created. If cleared before the decoder

    // creation promise is resolved; it indicates that Shutdown() was called and

    // further processing such as initialization should stop.

    RefPtr<DecoderCancelled> mLiveToken;

    RefPtr<CreateDecoderPromise> mCreateDecoderPromise;

    MozPromiseRequestHolder<InitPromise> mInitRequest;

  } mAudio, mVideo;

  void RunStage(Data& aData);

  void DoCreateDecoder(Data& aData);

  void DoInitDecoder(Data& aData);

  // guaranteed to be valid by the owner.

  const NotNull<MediaFormatReader*> mOwner;

};

void MediaFormatReader::DecoderFactory::CreateDecoder(TrackType aTrack) {

  MOZ_ASSERT(aTrack == TrackInfo::kAudioTrack ||

             aTrack == TrackInfo::kVideoTrack);

  Data& data = aTrack == TrackInfo::kAudioTrack ? mAudio : mVideo;

  MOZ_DIAGNOSTIC_ASSERT_IF(mOwner->GetDecoderData(data.mTrack).IsEncrypted(),

                           mOwner->mCDMProxy);

  RunStage(data);

void MediaFormatReader::DecoderFactory::RunStage(Data& aData) {

  switch (aData.mStage) {

    case Stage::None: {

      MOZ_DIAGNOSTIC_ASSERT(!aData.mToken);

      aData.mPolicy->Alloc()

          ->Then(

              mOwner->OwnerThread(), __func__,

              [this, &aData](RefPtr<Token> aToken) {

                aData.mTokenRequest.Complete();

                aData.mToken = std::move(aToken);

                aData.mStage = Stage::CreateDecoder;

                RunStage(aData);

},

              [&aData]() {

                aData.mTokenRequest.Complete();

                aData.mStage = Stage::None;

})

          ->Track(aData.mTokenRequest);

      aData.mStage = Stage::WaitForToken;

      break;

    case Stage::WaitForToken: {

      MOZ_DIAGNOSTIC_ASSERT(!aData.mToken);

      MOZ_DIAGNOSTIC_ASSERT(aData.mTokenRequest.Exists());

      break;

    case Stage::CreateDecoder: {

      MOZ_DIAGNOSTIC_ASSERT(aData.mToken);

      MOZ_DIAGNOSTIC_ASSERT(!aData.mDecoder);

      MOZ_DIAGNOSTIC_ASSERT(!aData.mInitRequest.Exists());

      DoCreateDecoder(aData);

      aData.mStage = Stage::WaitForInit;

      break;

    case Stage::WaitForInit: {

      MOZ_DIAGNOSTIC_ASSERT((aData.mDecoder && aData.mInitRequest.Exists()) ||

                            aData.mLiveToken);

      break;

void MediaFormatReader::DecoderFactory::DoCreateDecoder(Data& aData) {

  AUTO_PROFILER_LABEL("DecoderFactory::DoCreateDecoder", MEDIA_PLAYBACK);

  auto& ownerData = aData.mOwnerData;

  auto& decoder = mOwner->GetDecoderData(aData.mTrack);

  RefPtr<PDMFactory> platform = new PDMFactory();

  if (decoder.IsEncrypted()) {

    MOZ_DIAGNOSTIC_ASSERT(mOwner->mCDMProxy);

    platform->SetCDMProxy(mOwner->mCDMProxy);

  RefPtr<PlatformDecoderModule::CreateDecoderPromise> p;

  MediaFormatReader* owner = mOwner;

  auto onWaitingForKeyEvent =

      [owner = ThreadSafeWeakPtr<MediaFormatReader>(owner)]() {

        RefPtr<MediaFormatReader> mfr(owner);

        MOZ_DIAGNOSTIC_ASSERT(mfr, "The MediaFormatReader didn't wait for us");

        return mfr ? &mfr->OnTrackWaitingForKeyProducer() : nullptr;

};

  switch (aData.mTrack) {

    case TrackInfo::kAudioTrack: {

      p = platform->CreateDecoder(

          {*ownerData.GetCurrentInfo()->GetAsAudioInfo(), mOwner->mCrashHelper,

           CreateDecoderParams::UseNullDecoder(ownerData.mIsNullDecode),

           TrackInfo::kAudioTrack, std::move(onWaitingForKeyEvent),

           mOwner->mMediaEngineId, mOwner->mTrackingId,

           mOwner->mEncryptedCustomIdent

               ? CreateDecoderParams::EncryptedCustomIdent::True

               : CreateDecoderParams::EncryptedCustomIdent::False});

      break;

    case TrackType::kVideoTrack: {

      // Decoders use the layers backend to decide if they can use hardware

      // decoding, so specify LAYERS_NONE if we want to forcibly disable it.

      using Option = CreateDecoderParams::Option;

      using OptionSet = CreateDecoderParams::OptionSet;

      p = platform->CreateDecoder(

          {*ownerData.GetCurrentInfo()->GetAsVideoInfo(),

           mOwner->mKnowsCompositor, mOwner->GetImageContainer(),

           mOwner->mCrashHelper,

           CreateDecoderParams::UseNullDecoder(ownerData.mIsNullDecode),

           TrackType::kVideoTrack, std::move(onWaitingForKeyEvent),

           CreateDecoderParams::VideoFrameRate(ownerData.mMeanRate.Mean()),

           OptionSet(ownerData.mHardwareDecodingDisabled

                         ? Option::HardwareDecoderNotAllowed

                         : Option::Default),

           mOwner->mMediaEngineId, mOwner->mTrackingId,

           mOwner->mEncryptedCustomIdent

               ? CreateDecoderParams::EncryptedCustomIdent::True

               : CreateDecoderParams::EncryptedCustomIdent::False});

      break;

    default:

      p = PlatformDecoderModule::CreateDecoderPromise::CreateAndReject(

          NS_ERROR_DOM_MEDIA_FATAL_ERR, __func__);

  aData.mLiveToken = MakeRefPtr<Data::DecoderCancelled>();

  aData.mCreateDecoderPromise = p->Then(

      mOwner->OwnerThread(), __func__,

      [this, &aData, &ownerData, live = WeakPtr{aData.mLiveToken},

       owner = ThreadSafeWeakPtr<MediaFormatReader>(owner)](

          RefPtr<MediaDataDecoder>&& aDecoder) {

        if (!live) {

          return CreateDecoderPromise::CreateAndResolve(std::move(aDecoder),

                                                        __func__);

        aData.mLiveToken = nullptr;

        aData.mDecoder = new MediaDataDecoderProxy(

            aDecoder.forget(), do_AddRef(ownerData.mTaskQueue.get()));

        aData.mDecoder = new AllocationWrapper(aData.mDecoder.forget(),

                                               aData.mToken.forget());

        DecoderDoctorLogger::LinkParentAndChild(

            aData.mDecoder.get(), "decoder",

            "MediaFormatReader::DecoderFactory", this);

        DoInitDecoder(aData);

        return CreateDecoderPromise::CreateAndResolve(aData.mDecoder, __func__);

},

      [this, &aData,

       live = WeakPtr{aData.mLiveToken}](const MediaResult& aError) {

        NS_WARNING("Error constructing decoders");

        if (!live) {

          return CreateDecoderPromise::CreateAndReject(aError, __func__);

        aData.mLiveToken = nullptr;

        aData.mToken = nullptr;

        aData.mStage = Stage::None;

        aData.mOwnerData.mDescription = aError.Description();

        DDLOGEX2("MediaFormatReader::DecoderFactory", this, DDLogCategory::Log,

                 "create_decoder_error", aError);

        mOwner->NotifyError(aData.mTrack, aError);

        return CreateDecoderPromise::CreateAndReject(aError, __func__);

});

void MediaFormatReader::DecoderFactory::DoInitDecoder(Data& aData) {

  AUTO_PROFILER_LABEL("DecoderFactory::DoInitDecoder", MEDIA_PLAYBACK);

  auto& ownerData = aData.mOwnerData;

  DDLOGEX2("MediaFormatReader::DecoderFactory", this, DDLogCategory::Log,

           "initialize_decoder", DDNoValue{});

  aData.mDecoder->Init()

      ->Then(

          mOwner->OwnerThread(), __func__,

          [this, &aData, &ownerData](TrackType aTrack) {

            AUTO_PROFILER_LABEL("DecoderFactory::DoInitDecoder:Resolved",

                                MEDIA_PLAYBACK);

            aData.mInitRequest.Complete();

            aData.mStage = Stage::None;

            MutexAutoLock lock(ownerData.mMutex);

            ownerData.mDecoder = std::move(aData.mDecoder);

            ownerData.mDescription = ownerData.mDecoder->GetDescriptionName();

            DDLOGEX2("MediaFormatReader::DecoderFactory", this,

                     DDLogCategory::Log, "decoder_initialized", DDNoValue{});

            DecoderDoctorLogger::LinkParentAndChild(

                "MediaFormatReader::DecoderData", &ownerData, "decoder",

                ownerData.mDecoder.get());

            mOwner->SetVideoDecodeThreshold();

            mOwner->ScheduleUpdate(aTrack);

            if (aTrack == TrackInfo::kVideoTrack) {

              DecoderBenchmark::CheckVersion(

                  ownerData.GetCurrentInfo()->mMimeType);

            if (aTrack == TrackInfo::kAudioTrack) {

              ownerData.mProcessName = ownerData.mDecoder->GetProcessName();

              ownerData.mCodecName = ownerData.mDecoder->GetCodecName();

},

          [this, &aData, &ownerData](const MediaResult& aError) {

            AUTO_PROFILER_LABEL("DecoderFactory::DoInitDecoder:Rejected",

                                MEDIA_PLAYBACK);

            aData.mInitRequest.Complete();

            MOZ_RELEASE_ASSERT(!ownerData.mDecoder,

                               "Can't have a decoder already set");

            aData.mStage = Stage::None;

            mOwner->mShutdownPromisePool->ShutdownDecoder(

                aData.mDecoder.forget());

            DDLOGEX2("MediaFormatReader::DecoderFactory", this,

                     DDLogCategory::Log, "initialize_decoder_error", aError);

            mOwner->NotifyError(aData.mTrack, aError);

})

      ->Track(aData.mInitRequest);

// DemuxerProxy ensures that the original main demuxer is only ever accessed

// via its own dedicated task queue.

// This ensure that the reader's taskqueue will never blocked while a demuxer

// is itself blocked attempting to access the MediaCache or the MediaResource.

class MediaFormatReader::DemuxerProxy {

  using TrackType = TrackInfo::TrackType;

  class Wrapper;

 public:

  explicit DemuxerProxy(MediaDataDemuxer* aDemuxer)

      : mTaskQueue(TaskQueue::Create(

            GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER),

            "DemuxerProxy::mTaskQueue")),

        mData(new Data(aDemuxer)) {

    MOZ_COUNT_CTOR(DemuxerProxy);

  MOZ_COUNTED_DTOR(DemuxerProxy)

  RefPtr<ShutdownPromise> Shutdown() {

    RefPtr<Data> data = std::move(mData);

    return InvokeAsync(mTaskQueue, __func__, [data]() {

      // We need to clear our reference to the demuxer now. So that in the event

      // the init promise wasn't resolved, such as what can happen with the

      // mediasource demuxer that is waiting on more data, it will force the

      // init promise to be rejected.

      data->mDemuxer = nullptr;

      data->mAudioDemuxer = nullptr;

      data->mVideoDemuxer = nullptr;

      return ShutdownPromise::CreateAndResolve(true, __func__);

});

  RefPtr<MediaDataDemuxer::InitPromise> Init();

  Wrapper* GetTrackDemuxer(TrackType aTrack, uint32_t aTrackNumber) {

    MOZ_RELEASE_ASSERT(mData && mData->mInitDone);

    switch (aTrack) {

      case TrackInfo::kAudioTrack:

        return mData->mAudioDemuxer;

      case TrackInfo::kVideoTrack:

        return mData->mVideoDemuxer;

      default:

        return nullptr;

  uint32_t GetNumberTracks(TrackType aTrack) const {

    MOZ_RELEASE_ASSERT(mData && mData->mInitDone);

    switch (aTrack) {

      case TrackInfo::kAudioTrack:

        return mData->mNumAudioTrack;

      case TrackInfo::kVideoTrack:

        return mData->mNumVideoTrack;

      default:

        return 0;

  bool IsSeekable() const {

    MOZ_RELEASE_ASSERT(mData && mData->mInitDone);

    return mData->mSeekable;

  bool IsSeekableOnlyInBufferedRanges() const {

    MOZ_RELEASE_ASSERT(mData && mData->mInitDone);

    return mData->mSeekableOnlyInBufferedRange;

  UniquePtr<EncryptionInfo> GetCrypto() const {

    MOZ_RELEASE_ASSERT(mData && mData->mInitDone);

    if (!mData->mCrypto) {

      return nullptr;

    auto crypto = MakeUnique<EncryptionInfo>();

    *crypto = *mData->mCrypto;

    return crypto;

  RefPtr<NotifyDataArrivedPromise> NotifyDataArrived();

  bool ShouldComputeStartTime() const {

    MOZ_RELEASE_ASSERT(mData && mData->mInitDone);

    return mData->mShouldComputeStartTime;

 private:

  const RefPtr<TaskQueue> mTaskQueue;

  struct Data {

    NS_INLINE_DECL_THREADSAFE_REFCOUNTING(Data)

    explicit Data(MediaDataDemuxer* aDemuxer)

        : mInitDone(false), mDemuxer(aDemuxer) {}

    Atomic<bool> mInitDone;

    // Only ever accessed over mTaskQueue once.

    RefPtr<MediaDataDemuxer> mDemuxer;

    // Only accessed once InitPromise has been resolved and immutable after.

    // So we can safely access them without the use of the mutex.

    uint32_t mNumAudioTrack = 0;

    RefPtr<Wrapper> mAudioDemuxer;

    uint32_t mNumVideoTrack = 0;

    RefPtr<Wrapper> mVideoDemuxer;

    bool mSeekable = false;

    bool mSeekableOnlyInBufferedRange = false;

    bool mShouldComputeStartTime = true;

    UniquePtr<EncryptionInfo> mCrypto;

   private:

    ~Data() = default;

};

  RefPtr<Data> mData;

};

class MediaFormatReader::DemuxerProxy::Wrapper : public MediaTrackDemuxer {

 public:

  Wrapper(MediaTrackDemuxer* aTrackDemuxer, TaskQueue* aTaskQueue)

      : mMutex("TrackDemuxer Mutex"),

        mTaskQueue(aTaskQueue),

        mGetSamplesMayBlock(aTrackDemuxer->GetSamplesMayBlock()),

        mInfo(aTrackDemuxer->GetInfo()),

        mTrackDemuxer(aTrackDemuxer) {

    DecoderDoctorLogger::LogConstructionAndBase(

        "MediaFormatReader::DemuxerProxy::Wrapper", this,

        static_cast<const MediaTrackDemuxer*>(this));

    DecoderDoctorLogger::LinkParentAndChild(

        "MediaFormatReader::DemuxerProxy::Wrapper", this, "track demuxer",

        aTrackDemuxer);

  UniquePtr<TrackInfo> GetInfo() const override {

    if (!mInfo) {

      return nullptr;

    return mInfo->Clone();

  RefPtr<SeekPromise> Seek(const TimeUnit& aTime) override {

    RefPtr<Wrapper> self = this;

    return InvokeAsync(

               mTaskQueue, __func__,

               [self, aTime]() { return self->mTrackDemuxer->Seek(aTime); })

        ->Then(

            mTaskQueue, __func__,

            [self](const TimeUnit& aTime) {

              self->UpdateRandomAccessPoint();

              return SeekPromise::CreateAndResolve(aTime, __func__);

},

            [self](const MediaResult& aError) {

              self->UpdateRandomAccessPoint();

              return SeekPromise::CreateAndReject(aError, __func__);

});

  RefPtr<SamplesPromise> GetSamples(int32_t aNumSamples) override {

    RefPtr<Wrapper> self = this;

    return InvokeAsync(mTaskQueue, __func__,

                       [self, aNumSamples]() {

                         return self->mTrackDemuxer->GetSamples(aNumSamples);

})

        ->Then(

            mTaskQueue, __func__,

            [self](RefPtr<SamplesHolder> aSamples) {

              self->UpdateRandomAccessPoint();

              return SamplesPromise::CreateAndResolve(aSamples.forget(),

                                                      __func__);

},

            [self](const MediaResult& aError) {

              self->UpdateRandomAccessPoint();

              return SamplesPromise::CreateAndReject(aError, __func__);

});

  bool GetSamplesMayBlock() const override { return mGetSamplesMayBlock; }

  void Reset() override {

    RefPtr<Wrapper> self = this;

    nsresult rv = mTaskQueue->Dispatch(NS_NewRunnableFunction(

        "MediaFormatReader::DemuxerProxy::Wrapper::Reset",

        [self]() { self->mTrackDemuxer->Reset(); }));

    MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));

    Unused << rv;

  nsresult GetNextRandomAccessPoint(TimeUnit* aTime) override {

    MutexAutoLock lock(mMutex);

    if (NS_SUCCEEDED(mNextRandomAccessPointResult)) {

      *aTime = mNextRandomAccessPoint;

    return mNextRandomAccessPointResult;

  RefPtr<SkipAccessPointPromise> SkipToNextRandomAccessPoint(

      const TimeUnit& aTimeThreshold) override {

    RefPtr<Wrapper> self = this;

    return InvokeAsync(

               mTaskQueue, __func__,

               [self, aTimeThreshold]() {

                 return self->mTrackDemuxer->SkipToNextRandomAccessPoint(

                     aTimeThreshold);

})

        ->Then(

            mTaskQueue, __func__,

            [self](uint32_t aVal) {

              self->UpdateRandomAccessPoint();

              return SkipAccessPointPromise::CreateAndResolve(aVal, __func__);

},

            [self](const SkipFailureHolder& aError) {

              self->UpdateRandomAccessPoint();

              return SkipAccessPointPromise::CreateAndReject(aError, __func__);

});

  TimeIntervals GetBuffered() override {

    MutexAutoLock lock(mMutex);

    return mBuffered;

  void BreakCycles() override {}

 private:

  Mutex mMutex MOZ_UNANNOTATED;

  const RefPtr<TaskQueue> mTaskQueue;

  const bool mGetSamplesMayBlock;

  const UniquePtr<TrackInfo> mInfo;

  // mTrackDemuxer is only ever accessed on demuxer's task queue.

  RefPtr<MediaTrackDemuxer> mTrackDemuxer;

  // All following members are protected by mMutex

  nsresult mNextRandomAccessPointResult = NS_OK;

  TimeUnit mNextRandomAccessPoint;

  TimeIntervals mBuffered;

  friend class DemuxerProxy;

  ~Wrapper() {

    RefPtr<MediaTrackDemuxer> trackDemuxer = std::move(mTrackDemuxer);

    nsresult rv = mTaskQueue->Dispatch(NS_NewRunnableFunction(

        "MediaFormatReader::DemuxerProxy::Wrapper::~Wrapper",

        [trackDemuxer]() { trackDemuxer->BreakCycles(); }));

    MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));

    Unused << rv;

    DecoderDoctorLogger::LogDestruction(

        "MediaFormatReader::DemuxerProxy::Wrapper", this);

  void UpdateRandomAccessPoint() {

    MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn());

    if (!mTrackDemuxer) {

      // Detached.

      return;

    MutexAutoLock lock(mMutex);

    mNextRandomAccessPointResult =

        mTrackDemuxer->GetNextRandomAccessPoint(&mNextRandomAccessPoint);

  void UpdateBuffered() {

    MOZ_ASSERT(mTaskQueue->IsCurrentThreadIn());

    if (!mTrackDemuxer) {

      // Detached.

      return;

    MutexAutoLock lock(mMutex);

    mBuffered = mTrackDemuxer->GetBuffered();

};

RefPtr<MediaDataDemuxer::InitPromise> MediaFormatReader::DemuxerProxy::Init() {

  AUTO_PROFILER_LABEL("DemuxerProxy::Init", MEDIA_PLAYBACK);

  using InitPromise = MediaDataDemuxer::InitPromise;

  RefPtr<Data> data = mData;

  RefPtr<TaskQueue> taskQueue = mTaskQueue;

  return InvokeAsync(mTaskQueue, __func__,

                     [data, taskQueue]() {

                       if (!data->mDemuxer) {

                         return InitPromise::CreateAndReject(

                             NS_ERROR_DOM_MEDIA_CANCELED, __func__);

                       return data->mDemuxer->Init();

})

      ->Then(

          taskQueue, __func__,

          [data, taskQueue]() {

            AUTO_PROFILER_LABEL("DemuxerProxy::Init:Resolved", MEDIA_PLAYBACK);

            if (!data->mDemuxer) {  // Was shutdown.

              return InitPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_CANCELED,

                                                  __func__);

            data->mNumAudioTrack =

                data->mDemuxer->GetNumberTracks(TrackInfo::kAudioTrack);

            if (data->mNumAudioTrack) {

              RefPtr<MediaTrackDemuxer> d =

                  data->mDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0);

              if (d) {

                RefPtr<Wrapper> wrapper =

                    new DemuxerProxy::Wrapper(d, taskQueue);

                wrapper->UpdateBuffered();

                data->mAudioDemuxer = wrapper;

                DecoderDoctorLogger::LinkParentAndChild(

                    data->mDemuxer.get(), "decoder factory wrapper",

                    "MediaFormatReader::DecoderFactory::Wrapper",

                    wrapper.get());

            data->mNumVideoTrack =

                data->mDemuxer->GetNumberTracks(TrackInfo::kVideoTrack);

            if (data->mNumVideoTrack) {

              RefPtr<MediaTrackDemuxer> d =

                  data->mDemuxer->GetTrackDemuxer(TrackInfo::kVideoTrack, 0);

              if (d) {

                RefPtr<Wrapper> wrapper =

                    new DemuxerProxy::Wrapper(d, taskQueue);

                wrapper->UpdateBuffered();

                data->mVideoDemuxer = wrapper;

                DecoderDoctorLogger::LinkParentAndChild(

                    data->mDemuxer.get(), "decoder factory wrapper",

                    "MediaFormatReader::DecoderFactory::Wrapper",

                    wrapper.get());

            data->mCrypto = data->mDemuxer->GetCrypto();

            data->mSeekable = data->mDemuxer->IsSeekable();

            data->mSeekableOnlyInBufferedRange =

                data->mDemuxer->IsSeekableOnlyInBufferedRanges();

            data->mShouldComputeStartTime =

                data->mDemuxer->ShouldComputeStartTime();

            data->mInitDone = true;

            return InitPromise::CreateAndResolve(NS_OK, __func__);

},

          [](const MediaResult& aError) {

            return InitPromise::CreateAndReject(aError, __func__);

});

RefPtr<MediaFormatReader::NotifyDataArrivedPromise>

MediaFormatReader::DemuxerProxy::NotifyDataArrived() {

  RefPtr<Data> data = mData;

  return InvokeAsync(mTaskQueue, __func__, [data]() {

    if (!data->mDemuxer) {

      // Was shutdown.

      return NotifyDataArrivedPromise::CreateAndReject(

          NS_ERROR_DOM_MEDIA_CANCELED, __func__);

    data->mDemuxer->NotifyDataArrived();

    if (data->mAudioDemuxer) {

      data->mAudioDemuxer->UpdateBuffered();

    if (data->mVideoDemuxer) {

      data->mVideoDemuxer->UpdateBuffered();

    return NotifyDataArrivedPromise::CreateAndResolve(true, __func__);

});

MediaFormatReader::MediaFormatReader(MediaFormatReaderInit& aInit,

                                     MediaDataDemuxer* aDemuxer)

    : mTaskQueue(

          TaskQueue::Create(GetMediaThreadPool(MediaThreadType::SUPERVISOR),

                            "MediaFormatReader::mTaskQueue",

                            /* aSupportsTailDispatch = */ true)),

      mAudio(this, MediaData::Type::AUDIO_DATA,

             StaticPrefs::media_audio_max_decode_error()),

      mVideo(this, MediaData::Type::VIDEO_DATA,

             StaticPrefs::media_video_max_decode_error()),

      mWorkingInfoChanged(false, "MediaFormatReader::mWorkingInfoChanged"),

      mWatchManager(this, OwnerThread()),

      mIsWatchingWorkingInfo(false),

      mDemuxer(new DemuxerProxy(aDemuxer)),

      mDemuxerInitDone(false),

      mPendingNotifyDataArrived(false),

      mLastReportedNumDecodedFrames(0),

      mPreviousDecodedKeyframeTime_us(sNoPreviousDecodedKeyframe),

      mKnowsCompositor(aInit.mKnowsCompositor),

      mInitDone(false),

      mTrackDemuxersMayBlock(false),

      mSeekScheduled(false),

      mVideoFrameContainer(aInit.mVideoFrameContainer),

      mCrashHelper(aInit.mCrashHelper),

      mDecoderFactory(new DecoderFactory(this)),

      mShutdownPromisePool(new ShutdownPromisePool()),

      mBuffered(mTaskQueue, TimeIntervals(),

                "MediaFormatReader::mBuffered (Canonical)"),

      mFrameStats(aInit.mFrameStats),

      mMediaDecoderOwnerID(aInit.mMediaDecoderOwnerID),

      mTrackingId(std::move(aInit.mTrackingId)),

      mReadMetadataStartTime(Nothing()),

      mReadMetaDataTime(TimeDuration::Zero()),

      mTotalWaitingForVideoDataTime(TimeDuration::Zero()),

      mEncryptedCustomIdent(false) {

  MOZ_ASSERT(aDemuxer);

  MOZ_COUNT_CTOR(MediaFormatReader);

  DDLINKCHILD("audio decoder data", "MediaFormatReader::DecoderDataWithPromise",

              &mAudio);

  DDLINKCHILD("video decoder data", "MediaFormatReader::DecoderDataWithPromise",

              &mVideo);

  DDLINKCHILD("demuxer", aDemuxer);

  mOnTrackWaitingForKeyListener = OnTrackWaitingForKey().Connect(

      mTaskQueue, this, &MediaFormatReader::NotifyWaitingForKey);

MediaFormatReader::~MediaFormatReader() {

  MOZ_COUNT_DTOR(MediaFormatReader);

  MOZ_ASSERT(mShutdown);

RefPtr<ShutdownPromise> MediaFormatReader::Shutdown() {

  MOZ_ASSERT(OnTaskQueue());

  LOG("");

  mDemuxerInitRequest.DisconnectIfExists();

  mNotifyDataArrivedPromise.DisconnectIfExists();

  mMetadataPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);

  mSeekPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);

  mSkipRequest.DisconnectIfExists();

  mSetCDMPromise.RejectIfExists(

      MediaResult(NS_ERROR_DOM_INVALID_STATE_ERR,

                  "MediaFormatReader is shutting down"),

      __func__);

  if (mIsWatchingWorkingInfo) {

    mWatchManager.Unwatch(mWorkingInfoChanged,

                          &MediaFormatReader::NotifyTrackInfoUpdated);

  mWatchManager.Shutdown();

  if (mAudio.HasPromise()) {

    mAudio.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);

  if (mVideo.HasPromise()) {

    mVideo.RejectPromise(NS_ERROR_DOM_MEDIA_CANCELED, __func__);

  if (HasAudio()) {

    mAudio.ResetDemuxer();

    mAudio.mTrackDemuxer->BreakCycles();

      MutexAutoLock lock(mAudio.mMutex);

      mAudio.mTrackDemuxer = nullptr;

    mAudio.ResetState();

    ShutdownDecoder(TrackInfo::kAudioTrack);

  if (HasVideo()) {

    mVideo.ResetDemuxer();

    mVideo.mTrackDemuxer->BreakCycles();

      MutexAutoLock lock(mVideo.mMutex);

      mVideo.mTrackDemuxer = nullptr;

    mVideo.ResetState();

    ShutdownDecoder(TrackInfo::kVideoTrack);

  mShutdownPromisePool->Track(mDemuxer->Shutdown());

  mDemuxer = nullptr;

  mOnTrackWaitingForKeyListener.Disconnect();

  mShutdown = true;

  return mShutdownPromisePool->Shutdown()->Then(

      OwnerThread(), __func__, this, &MediaFormatReader::TearDownDecoders,

      &MediaFormatReader::TearDownDecoders);

void MediaFormatReader::ShutdownDecoder(TrackType aTrack) {

  LOGV("%s", TrackTypeToStr(aTrack));

  // Shut down the pending decoder if any.

  mDecoderFactory->ShutdownDecoder(aTrack);

  auto& decoder = GetDecoderData(aTrack);

  // Flush the decoder if necessary.

  decoder.Flush();

  // Shut down the decoder if any.

  decoder.ShutdownDecoder();

void MediaFormatReader::NotifyDecoderBenchmarkStore() {

  MOZ_ASSERT(OnTaskQueue());

  if (!StaticPrefs::media_mediacapabilities_from_database()) {

    return;

  auto& decoder = GetDecoderData(TrackInfo::kVideoTrack);

  if (decoder.GetCurrentInfo() && decoder.GetCurrentInfo()->GetAsVideoInfo()) {

    VideoInfo info = *(decoder.GetCurrentInfo()->GetAsVideoInfo());

    info.SetFrameRate(static_cast<int32_t>(ceil(decoder.mMeanRate.Mean())));

    mOnStoreDecoderBenchmark.Notify(std::move(info));

void MediaFormatReader::NotifyTrackInfoUpdated() {

  MOZ_ASSERT(OnTaskQueue());

  if (mWorkingInfoChanged) {

    mWorkingInfoChanged = false;

    VideoInfo videoInfo;

    AudioInfo audioInfo;

      MutexAutoLock lock(mVideo.mMutex);

      if (HasVideo()) {

        videoInfo = *mVideo.GetWorkingInfo()->GetAsVideoInfo();

      MutexAutoLock lock(mAudio.mMutex);

      if (HasAudio()) {

        audioInfo = *mAudio.GetWorkingInfo()->GetAsAudioInfo();

    mTrackInfoUpdatedEvent.Notify(videoInfo, audioInfo);

RefPtr<ShutdownPromise> MediaFormatReader::TearDownDecoders() {

  if (mAudio.mTaskQueue) {

    mAudio.mTaskQueue->BeginShutdown();

    mAudio.mTaskQueue->AwaitShutdownAndIdle();

    mAudio.mTaskQueue = nullptr;

  if (mVideo.mTaskQueue) {

    mVideo.mTaskQueue->BeginShutdown();

    mVideo.mTaskQueue->AwaitShutdownAndIdle();

    mVideo.mTaskQueue = nullptr;

  mDecoderFactory = nullptr;

  mVideoFrameContainer = nullptr;

  ReleaseResources();

  mBuffered.DisconnectAll();

  return mTaskQueue->BeginShutdown();

nsresult MediaFormatReader::Init() {

  MOZ_ASSERT(NS_IsMainThread(), "Must be on main thread.");

  mAudio.mTaskQueue =

      TaskQueue::Create(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER),

                        "MFR::mAudio::mTaskQueue");

  mVideo.mTaskQueue =

      TaskQueue::Create(GetMediaThreadPool(MediaThreadType::PLATFORM_DECODER),

                        "MFR::mVideo::mTaskQueue");

  return NS_OK;

bool MediaFormatReader::ResolveSetCDMPromiseIfDone(TrackType aTrack) {

  // When a CDM proxy is set, MFR would shutdown the existing MediaDataDecoder

  // and would create new one for specific track in the next Update.