Source code

Revision control

Copy as Markdown

Other Tools

/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set ts=2 sw=2 sts=2 et cindent: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "MediaDecoderStateMachine.h"
#include <algorithm>
#include <stdint.h>
#include <utility>
#include "AudioSegment.h"
#include "DOMMediaStream.h"
#include "ImageContainer.h"
#include "MediaDecoder.h"
#include "MediaShutdownManager.h"
#include "MediaTimer.h"
#include "MediaTrackGraph.h"
#include "PerformanceRecorder.h"
#include "ReaderProxy.h"
#include "TimeUnits.h"
#include "VideoSegment.h"
#include "VideoUtils.h"
#include "mediasink/AudioSink.h"
#include "mediasink/AudioSinkWrapper.h"
#include "mediasink/DecodedStream.h"
#include "mediasink/VideoSink.h"
#include "mozilla/Logging.h"
#include "mozilla/MathAlgorithms.h"
#include "mozilla/NotNull.h"
#include "mozilla/Preferences.h"
#include "mozilla/ProfilerLabels.h"
#include "mozilla/ProfilerMarkerTypes.h"
#include "mozilla/ProfilerMarkers.h"
#include "mozilla/SharedThreadPool.h"
#include "mozilla/Sprintf.h"
#include "mozilla/StaticPrefs_media.h"
#include "mozilla/TaskQueue.h"
#include "nsIMemoryReporter.h"
#include "nsPrintfCString.h"
#include "nsTArray.h"
namespace mozilla {
using namespace mozilla::media;
#define NS_DispatchToMainThread(...) \
CompileError_UseAbstractThreadDispatchInstead
// avoid redefined macro in unified build
#undef FMT
#undef LOG
#undef LOGV
#undef LOGW
#undef LOGE
#undef SFMT
#undef SLOG
#undef SLOGW
#undef SLOGE
#define FMT(x, ...) "Decoder=%p " x, mDecoderID, ##__VA_ARGS__
#define LOG(x, ...) \
DDMOZ_LOG(gMediaDecoderLog, LogLevel::Debug, "Decoder=%p " x, mDecoderID, \
##__VA_ARGS__)
#define LOGV(x, ...) \
DDMOZ_LOG(gMediaDecoderLog, LogLevel::Verbose, "Decoder=%p " x, mDecoderID, \
##__VA_ARGS__)
#define LOGW(x, ...) NS_WARNING(nsPrintfCString(FMT(x, ##__VA_ARGS__)).get())
#define LOGE(x, ...) \
NS_DebugBreak(NS_DEBUG_WARNING, \
nsPrintfCString(FMT(x, ##__VA_ARGS__)).get(), nullptr, \
__FILE__, __LINE__)
// Used by StateObject and its sub-classes
#define SFMT(x, ...) \
"Decoder=%p state=%s " x, mMaster->mDecoderID, ToStateStr(GetState()), \
##__VA_ARGS__
#define SLOG(x, ...) \
DDMOZ_LOGEX(mMaster, gMediaDecoderLog, LogLevel::Debug, "state=%s " x, \
ToStateStr(GetState()), ##__VA_ARGS__)
#define SLOGW(x, ...) NS_WARNING(nsPrintfCString(SFMT(x, ##__VA_ARGS__)).get())
#define SLOGE(x, ...) \
NS_DebugBreak(NS_DEBUG_WARNING, \
nsPrintfCString(SFMT(x, ##__VA_ARGS__)).get(), nullptr, \
__FILE__, __LINE__)
// Certain constants get stored as member variables and then adjusted by various
// scale factors on a per-decoder basis. We want to make sure to avoid using
// these constants directly, so we put them in a namespace.
namespace detail {
// Resume a suspended video decoder to the current playback position plus this
// time premium for compensating the seeking delay.
static constexpr auto RESUME_VIDEO_PREMIUM = TimeUnit::FromMicroseconds(125000);
static const int64_t AMPLE_AUDIO_USECS = 2000000;
// If more than this much decoded audio is queued, we'll hold off
// decoding more audio.
static constexpr auto AMPLE_AUDIO_THRESHOLD =
TimeUnit::FromMicroseconds(AMPLE_AUDIO_USECS);
} // namespace detail
// If we have fewer than LOW_VIDEO_FRAMES decoded frames, and
// we're not "prerolling video", we'll skip the video up to the next keyframe
// which is at or after the current playback position.
static const uint32_t LOW_VIDEO_FRAMES = 2;
// Arbitrary "frame duration" when playing only audio.
static const uint32_t AUDIO_DURATION_USECS = 40000;
namespace detail {
// If we have less than this much buffered data available, we'll consider
// ourselves to be running low on buffered data. We determine how much
// buffered data we have remaining using the reader's GetBuffered()
// implementation.
static const int64_t LOW_BUFFER_THRESHOLD_USECS = 5000000;
static constexpr auto LOW_BUFFER_THRESHOLD =
TimeUnit::FromMicroseconds(LOW_BUFFER_THRESHOLD_USECS);
// LOW_BUFFER_THRESHOLD_USECS needs to be greater than AMPLE_AUDIO_USECS,
// otherwise the skip-to-keyframe logic can activate when we're running low on
// data.
static_assert(LOW_BUFFER_THRESHOLD_USECS > AMPLE_AUDIO_USECS,
"LOW_BUFFER_THRESHOLD_USECS is too small");
} // namespace detail
// Amount of excess data to add in to the "should we buffer" calculation.
static constexpr auto EXHAUSTED_DATA_MARGIN =
TimeUnit::FromMicroseconds(100000);
static const uint32_t MIN_VIDEO_QUEUE_SIZE = 3;
static const uint32_t MAX_VIDEO_QUEUE_SIZE = 10;
#ifdef MOZ_APPLEMEDIA
static const uint32_t HW_VIDEO_QUEUE_SIZE = 10;
#else
static const uint32_t HW_VIDEO_QUEUE_SIZE = 3;
#endif
static const uint32_t VIDEO_QUEUE_SEND_TO_COMPOSITOR_SIZE = 9999;
static uint32_t sVideoQueueDefaultSize = MAX_VIDEO_QUEUE_SIZE;
static uint32_t sVideoQueueHWAccelSize = HW_VIDEO_QUEUE_SIZE;
static uint32_t sVideoQueueSendToCompositorSize =
VIDEO_QUEUE_SEND_TO_COMPOSITOR_SIZE;
static void InitVideoQueuePrefs() {
MOZ_ASSERT(NS_IsMainThread());
static bool sPrefInit = false;
if (!sPrefInit) {
sPrefInit = true;
sVideoQueueDefaultSize = Preferences::GetUint(
"media.video-queue.default-size", MAX_VIDEO_QUEUE_SIZE);
sVideoQueueHWAccelSize = Preferences::GetUint(
"media.video-queue.hw-accel-size", HW_VIDEO_QUEUE_SIZE);
sVideoQueueSendToCompositorSize =
Preferences::GetUint("media.video-queue.send-to-compositor-size",
VIDEO_QUEUE_SEND_TO_COMPOSITOR_SIZE);
}
}
template <typename Type, typename Function>
static void DiscardFramesFromTail(MediaQueue<Type>& aQueue,
const Function&& aTest) {
while (aQueue.GetSize()) {
if (aTest(aQueue.PeekBack()->mTime.ToMicroseconds())) {
RefPtr<Type> releaseMe = aQueue.PopBack();
continue;
}
break;
}
}
// Delay, in milliseconds, that tabs needs to be in background before video
// decoding is suspended.
static TimeDuration SuspendBackgroundVideoDelay() {
return TimeDuration::FromMilliseconds(
StaticPrefs::media_suspend_background_video_delay_ms());
}
class MediaDecoderStateMachine::StateObject {
public:
virtual ~StateObject() = default;
virtual void Exit() {} // Exit action.
virtual void Step() {} // Perform a 'cycle' of this state object.
virtual State GetState() const = 0;
// Event handlers for various events.
virtual void HandleAudioCaptured() {}
virtual void HandleAudioDecoded(AudioData* aAudio) {
Crash("Unexpected event!", __func__);
}
virtual void HandleVideoDecoded(VideoData* aVideo) {
Crash("Unexpected event!", __func__);
}
virtual void HandleAudioWaited(MediaData::Type aType) {
Crash("Unexpected event!", __func__);
}
virtual void HandleVideoWaited(MediaData::Type aType) {
Crash("Unexpected event!", __func__);
}
virtual void HandleWaitingForAudio() { Crash("Unexpected event!", __func__); }
virtual void HandleAudioCanceled() { Crash("Unexpected event!", __func__); }
virtual void HandleEndOfAudio() { Crash("Unexpected event!", __func__); }
virtual void HandleWaitingForVideo() { Crash("Unexpected event!", __func__); }
virtual void HandleVideoCanceled() { Crash("Unexpected event!", __func__); }
virtual void HandleEndOfVideo() { Crash("Unexpected event!", __func__); }
virtual RefPtr<MediaDecoder::SeekPromise> HandleSeek(
const SeekTarget& aTarget);
virtual RefPtr<ShutdownPromise> HandleShutdown();
virtual void HandleVideoSuspendTimeout() = 0;
virtual void HandleResumeVideoDecoding(const TimeUnit& aTarget);
virtual void HandlePlayStateChanged(MediaDecoder::PlayState aPlayState) {}
virtual void GetDebugInfo(
dom::MediaDecoderStateMachineDecodingStateDebugInfo& aInfo) {}
virtual void HandleLoopingChanged() {}
private:
template <class S, typename R, typename... As>
auto ReturnTypeHelper(R (S::*)(As...)) -> R;
void Crash(const char* aReason, const char* aSite) {
char buf[1024];
SprintfLiteral(buf, "%s state=%s callsite=%s", aReason,
ToStateStr(GetState()), aSite);
MOZ_ReportAssertionFailure(buf, __FILE__, __LINE__);
MOZ_CRASH();
}
protected:
enum class EventVisibility : int8_t { Observable, Suppressed };
using Master = MediaDecoderStateMachine;
explicit StateObject(Master* aPtr) : mMaster(aPtr) {}
TaskQueue* OwnerThread() const { return mMaster->mTaskQueue; }
ReaderProxy* Reader() const { return mMaster->mReader; }
const MediaInfo& Info() const { return mMaster->Info(); }
MediaQueue<AudioData>& AudioQueue() const { return mMaster->mAudioQueue; }
MediaQueue<VideoData>& VideoQueue() const { return mMaster->mVideoQueue; }
template <class S, typename... Args, size_t... Indexes>
auto CallEnterMemberFunction(S* aS, std::tuple<Args...>& aTuple,
std::index_sequence<Indexes...>)
-> decltype(ReturnTypeHelper(&S::Enter)) {
AUTO_PROFILER_LABEL("StateObject::CallEnterMemberFunction", MEDIA_PLAYBACK);
return aS->Enter(std::move(std::get<Indexes>(aTuple))...);
}
// Note this function will delete the current state object.
// Don't access members to avoid UAF after this call.
template <class S, typename... Ts>
auto SetState(Ts&&... aArgs) -> decltype(ReturnTypeHelper(&S::Enter)) {
// |aArgs| must be passed by reference to avoid passing MOZ_NON_PARAM class
// SeekJob by value. See bug 1287006 and bug 1338374. But we still *must*
// copy the parameters, because |Exit()| can modify them. See bug 1312321.
// So we 1) pass the parameters by reference, but then 2) immediately copy
// them into a Tuple to be safe against modification, and finally 3) move
// the elements of the Tuple into the final function call.
auto copiedArgs = std::make_tuple(std::forward<Ts>(aArgs)...);
// Copy mMaster which will reset to null.
auto* master = mMaster;
auto* s = new S(master);
// It's possible to seek again during seeking, otherwise the new state
// should always be different from the original one.
MOZ_ASSERT(GetState() != s->GetState() ||
GetState() == DECODER_STATE_SEEKING_ACCURATE ||
GetState() == DECODER_STATE_SEEKING_FROMDORMANT ||
GetState() == DECODER_STATE_SEEKING_NEXTFRAMESEEKING ||
GetState() == DECODER_STATE_SEEKING_VIDEOONLY);
SLOG("change state to: %s", ToStateStr(s->GetState()));
PROFILER_MARKER_TEXT("MDSM::StateChange", MEDIA_PLAYBACK, {},
nsPrintfCString("%s", ToStateStr(s->GetState())));
Exit();
// Delete the old state asynchronously to avoid UAF if the caller tries to
// access its members after SetState() returns.
master->OwnerThread()->DispatchDirectTask(
NS_NewRunnableFunction("MDSM::StateObject::DeleteOldState",
[toDelete = std::move(master->mStateObj)]() {}));
// Also reset mMaster to catch potentail UAF.
mMaster = nullptr;
master->mStateObj.reset(s);
return CallEnterMemberFunction(s, copiedArgs,
std::index_sequence_for<Ts...>{});
}
RefPtr<MediaDecoder::SeekPromise> SetSeekingState(
SeekJob&& aSeekJob, EventVisibility aVisibility);
void SetDecodingState();
// Take a raw pointer in order not to change the life cycle of MDSM.
// It is guaranteed to be valid by MDSM.
Master* mMaster;
};
/**
* Purpose: decode metadata like duration and dimensions of the media resource.
*
* Transition to other states when decoding metadata is done:
* SHUTDOWN if failing to decode metadata.
* DECODING_FIRSTFRAME otherwise.
*/
class MediaDecoderStateMachine::DecodeMetadataState
: public MediaDecoderStateMachine::StateObject {
public:
explicit DecodeMetadataState(Master* aPtr) : StateObject(aPtr) {}
void Enter() {
MOZ_ASSERT(!mMaster->mVideoDecodeSuspended);
MOZ_ASSERT(!mMetadataRequest.Exists());
SLOG("Dispatching AsyncReadMetadata");
// We disconnect mMetadataRequest in Exit() so it is fine to capture
// a raw pointer here.
Reader()
->ReadMetadata()
->Then(
OwnerThread(), __func__,
[this](MetadataHolder&& aMetadata) {
OnMetadataRead(std::move(aMetadata));
},
[this](const MediaResult& aError) { OnMetadataNotRead(aError); })
->Track(mMetadataRequest);
}
void Exit() override { mMetadataRequest.DisconnectIfExists(); }
State GetState() const override { return DECODER_STATE_DECODING_METADATA; }
RefPtr<MediaDecoder::SeekPromise> HandleSeek(
const SeekTarget& aTarget) override {
MOZ_DIAGNOSTIC_CRASH("Can't seek while decoding metadata.");
return MediaDecoder::SeekPromise::CreateAndReject(true, __func__);
}
void HandleVideoSuspendTimeout() override {
// Do nothing since no decoders are created yet.
}
void HandleResumeVideoDecoding(const TimeUnit&) override {
// We never suspend video decoding in this state.
MOZ_ASSERT(false, "Shouldn't have suspended video decoding.");
}
private:
void OnMetadataRead(MetadataHolder&& aMetadata);
void OnMetadataNotRead(const MediaResult& aError) {
AUTO_PROFILER_LABEL("DecodeMetadataState::OnMetadataNotRead",
MEDIA_PLAYBACK);
mMetadataRequest.Complete();
SLOGE("Decode metadata failed, shutting down decoder");
mMaster->DecodeError(aError);
}
MozPromiseRequestHolder<MediaFormatReader::MetadataPromise> mMetadataRequest;
};
/**
* Purpose: release decoder resources to save memory and hardware resources.
*
* Transition to:
* SEEKING if any seek request or play state changes to PLAYING.
*/
class MediaDecoderStateMachine::DormantState
: public MediaDecoderStateMachine::StateObject {
public:
explicit DormantState(Master* aPtr) : StateObject(aPtr) {}
void Enter() {
if (mMaster->IsPlaying()) {
mMaster->StopPlayback();
}
// Calculate the position to seek to when exiting dormant.
auto t = mMaster->mMediaSink->IsStarted() ? mMaster->GetClock()
: mMaster->GetMediaTime();
mMaster->AdjustByLooping(t);
mPendingSeek.mTarget.emplace(t, SeekTarget::Accurate);
// SeekJob asserts |mTarget.IsValid() == !mPromise.IsEmpty()| so we
// need to create the promise even it is not used at all.
// The promise may be used when coming out of DormantState into
// SeekingState.
RefPtr<MediaDecoder::SeekPromise> x =
mPendingSeek.mPromise.Ensure(__func__);
// Reset the decoding state to ensure that any queued video frames are
// released and don't consume video memory.
mMaster->ResetDecode();
// No need to call StopMediaSink() here.
// We will do it during seeking when exiting dormant.
// Ignore WAIT_FOR_DATA since we won't decode in dormant.
mMaster->mAudioWaitRequest.DisconnectIfExists();
mMaster->mVideoWaitRequest.DisconnectIfExists();
MaybeReleaseResources();
}
void Exit() override {
// mPendingSeek is either moved when exiting dormant or
// should be rejected here before transition to SHUTDOWN.
mPendingSeek.RejectIfExists(__func__);
}
State GetState() const override { return DECODER_STATE_DORMANT; }
RefPtr<MediaDecoder::SeekPromise> HandleSeek(
const SeekTarget& aTarget) override;
void HandleVideoSuspendTimeout() override {
// Do nothing since we've released decoders in Enter().
}
void HandleResumeVideoDecoding(const TimeUnit&) override {
// Do nothing since we won't resume decoding until exiting dormant.
}
void HandlePlayStateChanged(MediaDecoder::PlayState aPlayState) override;
void HandleAudioDecoded(AudioData*) override { MaybeReleaseResources(); }
void HandleVideoDecoded(VideoData*) override { MaybeReleaseResources(); }
void HandleWaitingForAudio() override { MaybeReleaseResources(); }
void HandleWaitingForVideo() override { MaybeReleaseResources(); }
void HandleAudioCanceled() override { MaybeReleaseResources(); }
void HandleVideoCanceled() override { MaybeReleaseResources(); }
void HandleEndOfAudio() override { MaybeReleaseResources(); }
void HandleEndOfVideo() override { MaybeReleaseResources(); }
private:
void MaybeReleaseResources() {
if (!mMaster->mAudioDataRequest.Exists() &&
!mMaster->mVideoDataRequest.Exists()) {
// Release decoders only when they are idle. Otherwise it might cause
// decode error later when resetting decoders during seeking.
mMaster->mReader->ReleaseResources();
}
}
SeekJob mPendingSeek;
};
/**
* Purpose: decode the 1st audio and video frames to fire the 'loadeddata'
* event.
*
* Transition to:
* SHUTDOWN if any decode error.
* SEEKING if any seek request.
* DECODING/LOOPING_DECODING when the 'loadeddata' event is fired.
*/
class MediaDecoderStateMachine::DecodingFirstFrameState
: public MediaDecoderStateMachine::StateObject {
public:
explicit DecodingFirstFrameState(Master* aPtr) : StateObject(aPtr) {}
void Enter();
void Exit() override {
// mPendingSeek is either moved in MaybeFinishDecodeFirstFrame()
// or should be rejected here before transition to SHUTDOWN.
mPendingSeek.RejectIfExists(__func__);
}
State GetState() const override { return DECODER_STATE_DECODING_FIRSTFRAME; }
void HandleAudioDecoded(AudioData* aAudio) override {
mMaster->PushAudio(aAudio);
MaybeFinishDecodeFirstFrame();
}
void HandleVideoDecoded(VideoData* aVideo) override {
mMaster->PushVideo(aVideo);
MaybeFinishDecodeFirstFrame();
}
void HandleWaitingForAudio() override {
mMaster->WaitForData(MediaData::Type::AUDIO_DATA);
}
void HandleAudioCanceled() override { mMaster->RequestAudioData(); }
void HandleEndOfAudio() override {
AudioQueue().Finish();
MaybeFinishDecodeFirstFrame();
}
void HandleWaitingForVideo() override {
mMaster->WaitForData(MediaData::Type::VIDEO_DATA);
}
void HandleVideoCanceled() override {
mMaster->RequestVideoData(media::TimeUnit());
}
void HandleEndOfVideo() override {
VideoQueue().Finish();
MaybeFinishDecodeFirstFrame();
}
void HandleAudioWaited(MediaData::Type aType) override {
mMaster->RequestAudioData();
}
void HandleVideoWaited(MediaData::Type aType) override {
mMaster->RequestVideoData(media::TimeUnit());
}
void HandleVideoSuspendTimeout() override {
// Do nothing for we need to decode the 1st video frame to get the
// dimensions.
}
void HandleResumeVideoDecoding(const TimeUnit&) override {
// We never suspend video decoding in this state.
MOZ_ASSERT(false, "Shouldn't have suspended video decoding.");
}
RefPtr<MediaDecoder::SeekPromise> HandleSeek(
const SeekTarget& aTarget) override {
if (mMaster->mIsMSE) {
return StateObject::HandleSeek(aTarget);
}
// Delay seek request until decoding first frames for non-MSE media.
SLOG("Not Enough Data to seek at this stage, queuing seek");
mPendingSeek.RejectIfExists(__func__);
mPendingSeek.mTarget.emplace(aTarget);
return mPendingSeek.mPromise.Ensure(__func__);
}
private:
// Notify FirstFrameLoaded if having decoded first frames and
// transition to SEEKING if there is any pending seek, or DECODING otherwise.
void MaybeFinishDecodeFirstFrame();
SeekJob mPendingSeek;
};
/**
* Purpose: decode audio/video data for playback.
*
* Transition to:
* DORMANT if playback is paused for a while.
* SEEKING if any seek request.
* SHUTDOWN if any decode error.
* BUFFERING if playback can't continue due to lack of decoded data.
* COMPLETED when having decoded all audio/video data.
* LOOPING_DECODING when media start seamless looping
*/
class MediaDecoderStateMachine::DecodingState
: public MediaDecoderStateMachine::StateObject {
public:
explicit DecodingState(Master* aPtr)
: StateObject(aPtr), mDormantTimer(OwnerThread()) {}
void Enter();
void Exit() override {
if (!mDecodeStartTime.IsNull()) {
TimeDuration decodeDuration = TimeStamp::Now() - mDecodeStartTime;
SLOG("Exiting DECODING, decoded for %.3lfs", decodeDuration.ToSeconds());
}
mDormantTimer.Reset();
mOnAudioPopped.DisconnectIfExists();
mOnVideoPopped.DisconnectIfExists();
}
void Step() override;
State GetState() const override { return DECODER_STATE_DECODING; }
void HandleAudioDecoded(AudioData* aAudio) override {
mMaster->PushAudio(aAudio);
DispatchDecodeTasksIfNeeded();
MaybeStopPrerolling();
}
void HandleVideoDecoded(VideoData* aVideo) override {
// We only do this check when we're not looping, which can be known by
// checking the queue's offset.
const auto currentTime = mMaster->GetMediaTime();
if (aVideo->GetEndTime() < currentTime &&
VideoQueue().GetOffset() == media::TimeUnit::Zero()) {
if (!mVideoFirstLateTime) {
mVideoFirstLateTime = Some(TimeStamp::Now());
}
PROFILER_MARKER("Video falling behind", MEDIA_PLAYBACK, {},
VideoFallingBehindMarker, aVideo->mTime.ToMicroseconds(),
currentTime.ToMicroseconds());
SLOG("video %" PRId64 " starts being late (current=%" PRId64 ")",
aVideo->mTime.ToMicroseconds(), currentTime.ToMicroseconds());
} else {
mVideoFirstLateTime.reset();
}
mMaster->PushVideo(aVideo);
DispatchDecodeTasksIfNeeded();
MaybeStopPrerolling();
}
void HandleAudioCanceled() override { mMaster->RequestAudioData(); }
void HandleVideoCanceled() override {
mMaster->RequestVideoData(mMaster->GetMediaTime(),
ShouldRequestNextKeyFrame());
}
void HandleEndOfAudio() override;
void HandleEndOfVideo() override;
void HandleWaitingForAudio() override {
mMaster->WaitForData(MediaData::Type::AUDIO_DATA);
MaybeStopPrerolling();
}
void HandleWaitingForVideo() override {
mMaster->WaitForData(MediaData::Type::VIDEO_DATA);
MaybeStopPrerolling();
}
void HandleAudioWaited(MediaData::Type aType) override {
mMaster->RequestAudioData();
}
void HandleVideoWaited(MediaData::Type aType) override {
mMaster->RequestVideoData(mMaster->GetMediaTime(),
ShouldRequestNextKeyFrame());
}
void HandleAudioCaptured() override {
MaybeStopPrerolling();
// MediaSink is changed. Schedule Step() to check if we can start playback.
mMaster->ScheduleStateMachine();
}
void HandleVideoSuspendTimeout() override {
// No video, so nothing to suspend.
if (!mMaster->HasVideo()) {
return;
}
PROFILER_MARKER_UNTYPED("MDSM::EnterVideoSuspend", MEDIA_PLAYBACK);
mMaster->mVideoDecodeSuspended = true;
mMaster->mOnPlaybackEvent.Notify(MediaPlaybackEvent::EnterVideoSuspend);
Reader()->SetVideoBlankDecode(true);
}
void HandlePlayStateChanged(MediaDecoder::PlayState aPlayState) override {
// Schedule Step() to check if we can start or stop playback.
mMaster->ScheduleStateMachine();
if (aPlayState == MediaDecoder::PLAY_STATE_PLAYING) {
// Try to dispatch decoding tasks for mMinimizePreroll might be reset.
DispatchDecodeTasksIfNeeded();
}
if (aPlayState == MediaDecoder::PLAY_STATE_PAUSED) {
StartDormantTimer();
mVideoFirstLateTime.reset();
} else {
mDormantTimer.Reset();
}
}
void GetDebugInfo(
dom::MediaDecoderStateMachineDecodingStateDebugInfo& aInfo) override {
aInfo.mIsPrerolling = mIsPrerolling;
}
void HandleLoopingChanged() override { SetDecodingState(); }
protected:
virtual void EnsureAudioDecodeTaskQueued();
virtual void EnsureVideoDecodeTaskQueued();
virtual bool ShouldStopPrerolling() const {
return mIsPrerolling &&
(DonePrerollingAudio() ||
IsWaitingData(MediaData::Type::AUDIO_DATA)) &&
(DonePrerollingVideo() ||
IsWaitingData(MediaData::Type::VIDEO_DATA));
}
virtual bool IsWaitingData(MediaData::Type aType) const {
if (aType == MediaData::Type::AUDIO_DATA) {
return mMaster->IsWaitingAudioData();
}
MOZ_ASSERT(aType == MediaData::Type::VIDEO_DATA);
return mMaster->IsWaitingVideoData();
}
void MaybeStopPrerolling() {
if (ShouldStopPrerolling()) {
mIsPrerolling = false;
// Check if we can start playback.
mMaster->ScheduleStateMachine();
}
}
bool ShouldRequestNextKeyFrame() const {
if (!mVideoFirstLateTime) {
return false;
}
const double elapsedTimeMs =
(TimeStamp::Now() - *mVideoFirstLateTime).ToMilliseconds();
const bool rv = elapsedTimeMs >=
StaticPrefs::media_decoder_skip_when_video_too_slow_ms();
if (rv) {
PROFILER_MARKER_UNTYPED("Skipping to next keyframe", MEDIA_PLAYBACK);
SLOG(
"video has been late behind media time for %f ms, should skip to "
"next key frame",
elapsedTimeMs);
}
return rv;
}
virtual bool IsBufferingAllowed() const { return true; }
private:
void DispatchDecodeTasksIfNeeded();
void MaybeStartBuffering();
// At the start of decoding we want to "preroll" the decode until we've
// got a few frames decoded before we consider whether decode is falling
// behind. Otherwise our "we're falling behind" logic will trigger
// unnecessarily if we start playing as soon as the first sample is
// decoded. These two fields store how many video frames and audio
// samples we must consume before are considered to be finished prerolling.
TimeUnit AudioPrerollThreshold() const {
return (mMaster->mAmpleAudioThreshold / 2)
.MultDouble(mMaster->mPlaybackRate);
}
uint32_t VideoPrerollFrames() const {
return std::min(
static_cast<uint32_t>(
mMaster->GetAmpleVideoFrames() / 2. * mMaster->mPlaybackRate + 1),
sVideoQueueDefaultSize);
}
bool DonePrerollingAudio() const {
return !mMaster->IsAudioDecoding() ||
mMaster->GetDecodedAudioDuration() >= AudioPrerollThreshold();
}
bool DonePrerollingVideo() const {
return !mMaster->IsVideoDecoding() ||
static_cast<uint32_t>(mMaster->VideoQueue().GetSize()) >=
VideoPrerollFrames();
}
void StartDormantTimer() {
if (!mMaster->mMediaSeekable) {
// Don't enter dormant if the media is not seekable because we need to
// seek when exiting dormant.
return;
}
auto timeout = StaticPrefs::media_dormant_on_pause_timeout_ms();
if (timeout < 0) {
// Disabled when timeout is negative.
return;
}
if (timeout == 0) {
// Enter dormant immediately without scheduling a timer.
SetState<DormantState>();
return;
}
if (mMaster->mMinimizePreroll) {
SetState<DormantState>();
return;
}
TimeStamp target =
TimeStamp::Now() + TimeDuration::FromMilliseconds(timeout);
mDormantTimer.Ensure(
target,
[this]() {
AUTO_PROFILER_LABEL("DecodingState::StartDormantTimer:SetDormant",
MEDIA_PLAYBACK);
mDormantTimer.CompleteRequest();
SetState<DormantState>();
},
[this]() { mDormantTimer.CompleteRequest(); });
}
// Time at which we started decoding.
TimeStamp mDecodeStartTime;
// When we start decoding (either for the first time, or after a pause)
// we may be low on decoded data. We don't want our "low data" logic to
// kick in and decide that we're low on decoded data because the download
// can't keep up with the decode, and cause us to pause playback. So we
// have a "preroll" stage, where we ignore the results of our "low data"
// logic during the first few frames of our decode. This occurs during
// playback.
bool mIsPrerolling = true;
// Fired when playback is paused for a while to enter dormant.
DelayedScheduler<TimeStamp> mDormantTimer;
MediaEventListener mOnAudioPopped;
MediaEventListener mOnVideoPopped;
// If video has been later than the media time, this will records when the
// video started being late. It will be reset once video catches up with the
// media time.
Maybe<TimeStamp> mVideoFirstLateTime;
};
/**
* Purpose: decode audio data for playback when media is in seamless
* looping, we will adjust media time to make samples time monotonically
* increasing. All its methods runs on its owner thread (MDSM thread).
*
* Transition to:
* DORMANT if playback is paused for a while.
* SEEKING if any seek request.
* SHUTDOWN if any decode error.
* BUFFERING if playback can't continue due to lack of decoded data.
* COMPLETED when the media resource is closed and no data is available
* anymore.
* DECODING when media stops seamless looping.
*/
class MediaDecoderStateMachine::LoopingDecodingState
: public MediaDecoderStateMachine::DecodingState {
public:
explicit LoopingDecodingState(Master* aPtr)
: DecodingState(aPtr),
mIsReachingAudioEOS(!mMaster->IsAudioDecoding()),
mIsReachingVideoEOS(!mMaster->IsVideoDecoding()),
mAudioEndedBeforeEnteringStateWithoutDuration(false),
mVideoEndedBeforeEnteringStateWithoutDuration(false) {
MOZ_ASSERT(mMaster->mLooping);
SLOG(
"LoopingDecodingState ctor, mIsReachingAudioEOS=%d, "
"mIsReachingVideoEOS=%d",
mIsReachingAudioEOS, mIsReachingVideoEOS);
// If the track has reached EOS and we already have its last data, then we
// can know its duration. But if playback starts from EOS (due to seeking),
// the decoded end time would be zero because none of data gets decoded yet.
if (mIsReachingAudioEOS) {
if (mMaster->HasLastDecodedData(MediaData::Type::AUDIO_DATA) &&
!mMaster->mAudioTrackDecodedDuration) {
mMaster->mAudioTrackDecodedDuration.emplace(
mMaster->mDecodedAudioEndTime);
SLOG("determine mAudioTrackDecodedDuration");
} else {
mAudioEndedBeforeEnteringStateWithoutDuration = true;
SLOG("still don't know mAudioTrackDecodedDuration");
}
}
if (mIsReachingVideoEOS) {
if (mMaster->HasLastDecodedData(MediaData::Type::VIDEO_DATA) &&
!mMaster->mVideoTrackDecodedDuration) {
mMaster->mVideoTrackDecodedDuration.emplace(
mMaster->mDecodedVideoEndTime);
SLOG("determine mVideoTrackDecodedDuration");
} else {
mVideoEndedBeforeEnteringStateWithoutDuration = true;
SLOG("still don't know mVideoTrackDecodedDuration");
}
}
// We might be able to determine the duration already, let's check.
if (mIsReachingAudioEOS || mIsReachingVideoEOS) {
Unused << DetermineOriginalDecodedDurationIfNeeded();
}
// If we've looped at least once before, then we need to update queue offset
// correctly to make the media data time and the clock time consistent.
// Otherwise, it would cause a/v desync.
if (mMaster->mOriginalDecodedDuration != media::TimeUnit::Zero()) {
if (mIsReachingAudioEOS && mMaster->HasAudio()) {
AudioQueue().SetOffset(AudioQueue().GetOffset() +
mMaster->mOriginalDecodedDuration);
}
if (mIsReachingVideoEOS && mMaster->HasVideo()) {
VideoQueue().SetOffset(VideoQueue().GetOffset() +
mMaster->mOriginalDecodedDuration);
}
}
}
void Enter() {
if (mMaster->HasAudio() && mIsReachingAudioEOS) {
SLOG("audio has ended, request the data again.");
RequestDataFromStartPosition(TrackInfo::TrackType::kAudioTrack);
}
if (mMaster->HasVideo() && mIsReachingVideoEOS) {
SLOG("video has ended, request the data again.");
RequestDataFromStartPosition(TrackInfo::TrackType::kVideoTrack);
}
DecodingState::Enter();
}
void Exit() override {
MOZ_DIAGNOSTIC_ASSERT(mMaster->OnTaskQueue());
SLOG("Leaving looping state, offset [a=%" PRId64 ",v=%" PRId64
"], endtime [a=%" PRId64 ",v=%" PRId64 "], track duration [a=%" PRId64
",v=%" PRId64 "], waiting=%s",
AudioQueue().GetOffset().ToMicroseconds(),
VideoQueue().GetOffset().ToMicroseconds(),
mMaster->mDecodedAudioEndTime.ToMicroseconds(),
mMaster->mDecodedVideoEndTime.ToMicroseconds(),
mMaster->mAudioTrackDecodedDuration
? mMaster->mAudioTrackDecodedDuration->ToMicroseconds()
: 0,
mMaster->mVideoTrackDecodedDuration
? mMaster->mVideoTrackDecodedDuration->ToMicroseconds()
: 0,
mDataWaitingTimestampAdjustment
? MediaData::EnumValueToString(
mDataWaitingTimestampAdjustment->mType)
: "none");
if (ShouldDiscardLoopedData(MediaData::Type::AUDIO_DATA)) {
DiscardLoopedData(MediaData::Type::AUDIO_DATA);
}
if (ShouldDiscardLoopedData(MediaData::Type::VIDEO_DATA)) {
DiscardLoopedData(MediaData::Type::VIDEO_DATA);
}
if (mMaster->HasAudio() && HasDecodedLastAudioFrame()) {
SLOG("Mark audio queue as finished");
mMaster->mAudioDataRequest.DisconnectIfExists();
mMaster->mAudioWaitRequest.DisconnectIfExists();
AudioQueue().Finish();
}
if (mMaster->HasVideo() && HasDecodedLastVideoFrame()) {
SLOG("Mark video queue as finished");
mMaster->mVideoDataRequest.DisconnectIfExists();
mMaster->mVideoWaitRequest.DisconnectIfExists();
VideoQueue().Finish();
}
// Clear waiting data should be done after marking queue as finished.
mDataWaitingTimestampAdjustment = nullptr;
mAudioDataRequest.DisconnectIfExists();
mVideoDataRequest.DisconnectIfExists();
mAudioSeekRequest.DisconnectIfExists();
mVideoSeekRequest.DisconnectIfExists();
DecodingState::Exit();
}
~LoopingDecodingState() {
MOZ_DIAGNOSTIC_ASSERT(!mAudioDataRequest.Exists());
MOZ_DIAGNOSTIC_ASSERT(!mVideoDataRequest.Exists());
MOZ_DIAGNOSTIC_ASSERT(!mAudioSeekRequest.Exists());
MOZ_DIAGNOSTIC_ASSERT(!mVideoSeekRequest.Exists());
}
State GetState() const override { return DECODER_STATE_LOOPING_DECODING; }
void HandleAudioDecoded(AudioData* aAudio) override {
// TODO : check if we need to update mOriginalDecodedDuration
// After pushing data to the queue, timestamp might be adjusted.
DecodingState::HandleAudioDecoded(aAudio);
mMaster->mDecodedAudioEndTime =
std::max(aAudio->GetEndTime(), mMaster->mDecodedAudioEndTime);
SLOG("audio sample after time-adjustment [%" PRId64 ",%" PRId64 "]",
aAudio->mTime.ToMicroseconds(), aAudio->GetEndTime().ToMicroseconds());
}
void HandleVideoDecoded(VideoData* aVideo) override {
// TODO : check if we need to update mOriginalDecodedDuration
// Here sample still keeps its original timestamp.
// This indicates there is a shorter audio track, and it's the first time in
// the looping (audio ends but video is playing) so that we haven't been
// able to determine the decoded duration. Therefore, we fill the gap
// between two tracks before video ends. Afterward, this adjustment will be
// done in `HandleEndOfAudio()`.
if (mMaster->mOriginalDecodedDuration == media::TimeUnit::Zero() &&
mMaster->mAudioTrackDecodedDuration &&
aVideo->GetEndTime() > *mMaster->mAudioTrackDecodedDuration) {
media::TimeUnit gap;
// First time we fill gap between the video frame to the last audio.
if (auto prevVideo = VideoQueue().PeekBack();
prevVideo &&
prevVideo->GetEndTime() < *mMaster->mAudioTrackDecodedDuration) {
gap =
aVideo->GetEndTime().ToBase(*mMaster->mAudioTrackDecodedDuration) -
*mMaster->mAudioTrackDecodedDuration;
}
// Then fill the gap for all following videos.
else {
gap = aVideo->mDuration.ToBase(*mMaster->mAudioTrackDecodedDuration);
}