Source code

Revision control

Other Tools

/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim:set ts=2 sw=2 sts=2 et cindent: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include <algorithm>
#include <stdint.h>
#include <utility>
#include "mediasink/AudioSink.h"
#include "mediasink/AudioSinkWrapper.h"
#include "mediasink/DecodedStream.h"
#include "mediasink/VideoSink.h"
#include "mozilla/Logging.h"
#include "mozilla/MathAlgorithms.h"
#include "mozilla/NotNull.h"
#include "mozilla/SharedThreadPool.h"
#include "mozilla/Sprintf.h"
#include "mozilla/StaticPrefs_media.h"
#include "mozilla/Telemetry.h"
#include "mozilla/TaskQueue.h"
#include "mozilla/Tuple.h"
#include "nsIMemoryReporter.h"
#include "nsPrintfCString.h"
#include "nsTArray.h"
#include "AudioSegment.h"
#include "DOMMediaStream.h"
#include "ImageContainer.h"
#include "MediaDecoder.h"
#include "MediaDecoderStateMachine.h"
#include "MediaShutdownManager.h"
#include "MediaTrackGraph.h"
#include "MediaTimer.h"
#include "ReaderProxy.h"
#include "TimeUnits.h"
#include "VideoSegment.h"
#include "VideoUtils.h"
#ifdef MOZ_GECKO_PROFILER
# include "ProfilerMarkerPayload.h"
# define MDSM_ERROR_MARKER(tag, error, markerTime) \
PROFILER_ADD_MARKER_WITH_PAYLOAD(tag, MEDIA_PLAYBACK, TextMarkerPayload, \
(error, markerTime))
# define MDSM_SAMPLE_MARKER(tag, startTime, endTime) \
PROFILER_ADD_MARKER_WITH_PAYLOAD( \
tag, MEDIA_PLAYBACK, MediaSampleMarkerPayload, (startTime, endTime))
#else
# define MDSM_ERROR_MARKER(tag, error, markerTime)
# define MDSM_SAMPLE_MARKER(tag, startTime, endTime)
#endif
namespace mozilla {
using namespace mozilla::media;
#define NS_DispatchToMainThread(...) \
CompileError_UseAbstractThreadDispatchInstead
// avoid redefined macro in unified build
#undef FMT
#undef LOG
#undef LOGV
#undef LOGW
#undef LOGE
#undef SFMT
#undef SLOG
#undef SLOGW
#undef SLOGE
#define FMT(x, ...) "Decoder=%p " x, mDecoderID, ##__VA_ARGS__
#define LOG(x, ...) \
DDMOZ_LOG(gMediaDecoderLog, LogLevel::Debug, "Decoder=%p " x, mDecoderID, \
##__VA_ARGS__)
#define LOGV(x, ...) \
DDMOZ_LOG(gMediaDecoderLog, LogLevel::Verbose, "Decoder=%p " x, mDecoderID, \
##__VA_ARGS__)
#define LOGW(x, ...) NS_WARNING(nsPrintfCString(FMT(x, ##__VA_ARGS__)).get())
#define LOGE(x, ...) \
NS_DebugBreak(NS_DEBUG_WARNING, \
nsPrintfCString(FMT(x, ##__VA_ARGS__)).get(), nullptr, \
__FILE__, __LINE__)
// Used by StateObject and its sub-classes
#define SFMT(x, ...) \
"Decoder=%p state=%s " x, mMaster->mDecoderID, ToStateStr(GetState()), \
##__VA_ARGS__
#define SLOG(x, ...) \
DDMOZ_LOGEX(mMaster, gMediaDecoderLog, LogLevel::Debug, "state=%s " x, \
ToStateStr(GetState()), ##__VA_ARGS__)
#define SLOGW(x, ...) NS_WARNING(nsPrintfCString(SFMT(x, ##__VA_ARGS__)).get())
#define SLOGE(x, ...) \
NS_DebugBreak(NS_DEBUG_WARNING, \
nsPrintfCString(SFMT(x, ##__VA_ARGS__)).get(), nullptr, \
__FILE__, __LINE__)
// Certain constants get stored as member variables and then adjusted by various
// scale factors on a per-decoder basis. We want to make sure to avoid using
// these constants directly, so we put them in a namespace.
namespace detail {
// Resume a suspended video decoder to the current playback position plus this
// time premium for compensating the seeking delay.
static constexpr auto RESUME_VIDEO_PREMIUM = TimeUnit::FromMicroseconds(125000);
static const int64_t AMPLE_AUDIO_USECS = 2000000;
// If more than this much decoded audio is queued, we'll hold off
// decoding more audio.
static constexpr auto AMPLE_AUDIO_THRESHOLD =
TimeUnit::FromMicroseconds(AMPLE_AUDIO_USECS);
} // namespace detail
// If we have fewer than LOW_VIDEO_FRAMES decoded frames, and
// we're not "prerolling video", we'll skip the video up to the next keyframe
// which is at or after the current playback position.
static const uint32_t LOW_VIDEO_FRAMES = 2;
// Arbitrary "frame duration" when playing only audio.
static const int AUDIO_DURATION_USECS = 40000;
namespace detail {
// If we have less than this much buffered data available, we'll consider
// ourselves to be running low on buffered data. We determine how much
// buffered data we have remaining using the reader's GetBuffered()
// implementation.
static const int64_t LOW_BUFFER_THRESHOLD_USECS = 5000000;
static constexpr auto LOW_BUFFER_THRESHOLD =
TimeUnit::FromMicroseconds(LOW_BUFFER_THRESHOLD_USECS);
// LOW_BUFFER_THRESHOLD_USECS needs to be greater than AMPLE_AUDIO_USECS,
// otherwise the skip-to-keyframe logic can activate when we're running low on
// data.
static_assert(LOW_BUFFER_THRESHOLD_USECS > AMPLE_AUDIO_USECS,
"LOW_BUFFER_THRESHOLD_USECS is too small");
} // namespace detail
// Amount of excess data to add in to the "should we buffer" calculation.
static constexpr auto EXHAUSTED_DATA_MARGIN =
TimeUnit::FromMicroseconds(100000);
static const uint32_t MIN_VIDEO_QUEUE_SIZE = 3;
static const uint32_t MAX_VIDEO_QUEUE_SIZE = 10;
#ifdef MOZ_APPLEMEDIA
static const uint32_t HW_VIDEO_QUEUE_SIZE = 10;
#else
static const uint32_t HW_VIDEO_QUEUE_SIZE = 3;
#endif
static const uint32_t VIDEO_QUEUE_SEND_TO_COMPOSITOR_SIZE = 9999;
static uint32_t sVideoQueueDefaultSize = MAX_VIDEO_QUEUE_SIZE;
static uint32_t sVideoQueueHWAccelSize = HW_VIDEO_QUEUE_SIZE;
static uint32_t sVideoQueueSendToCompositorSize =
VIDEO_QUEUE_SEND_TO_COMPOSITOR_SIZE;
static void InitVideoQueuePrefs() {
MOZ_ASSERT(NS_IsMainThread());
static bool sPrefInit = false;
if (!sPrefInit) {
sPrefInit = true;
sVideoQueueDefaultSize = Preferences::GetUint(
"media.video-queue.default-size", MAX_VIDEO_QUEUE_SIZE);
sVideoQueueHWAccelSize = Preferences::GetUint(
"media.video-queue.hw-accel-size", HW_VIDEO_QUEUE_SIZE);
sVideoQueueSendToCompositorSize =
Preferences::GetUint("media.video-queue.send-to-compositor-size",
VIDEO_QUEUE_SEND_TO_COMPOSITOR_SIZE);
}
}
template <typename Type, typename Function>
static void DiscardFramesFromTail(MediaQueue<Type>& aQueue,
const Function&& aTest) {
while (aQueue.GetSize()) {
if (aTest(aQueue.PeekBack()->mTime.ToMicroseconds())) {
RefPtr<Type> releaseMe = aQueue.PopBack();
continue;
}
break;
}
}
// Delay, in milliseconds, that tabs needs to be in background before video
// decoding is suspended.
static TimeDuration SuspendBackgroundVideoDelay() {
return TimeDuration::FromMilliseconds(
StaticPrefs::media_suspend_bkgnd_video_delay_ms());
}
class MediaDecoderStateMachine::StateObject {
public:
virtual ~StateObject() = default;
virtual void Exit() {} // Exit action.
virtual void Step() {} // Perform a 'cycle' of this state object.
virtual State GetState() const = 0;
// Event handlers for various events.
virtual void HandleAudioCaptured() {}
virtual void HandleAudioDecoded(AudioData* aAudio) {
Crash("Unexpected event!", __func__);
}
virtual void HandleVideoDecoded(VideoData* aVideo, TimeStamp aDecodeStart) {
Crash("Unexpected event!", __func__);
}
virtual void HandleAudioWaited(MediaData::Type aType) {
Crash("Unexpected event!", __func__);
}
virtual void HandleVideoWaited(MediaData::Type aType) {
Crash("Unexpected event!", __func__);
}
virtual void HandleWaitingForAudio() { Crash("Unexpected event!", __func__); }
virtual void HandleAudioCanceled() { Crash("Unexpected event!", __func__); }
virtual void HandleEndOfAudio() { Crash("Unexpected event!", __func__); }
virtual void HandleWaitingForVideo() { Crash("Unexpected event!", __func__); }
virtual void HandleVideoCanceled() { Crash("Unexpected event!", __func__); }
virtual void HandleEndOfVideo() { Crash("Unexpected event!", __func__); }
virtual RefPtr<MediaDecoder::SeekPromise> HandleSeek(SeekTarget aTarget);
virtual RefPtr<ShutdownPromise> HandleShutdown();
virtual void HandleVideoSuspendTimeout() = 0;
virtual void HandleResumeVideoDecoding(const TimeUnit& aTarget);
virtual void HandlePlayStateChanged(MediaDecoder::PlayState aPlayState) {}
virtual void GetDebugInfo(
dom::MediaDecoderStateMachineDecodingStateDebugInfo& aInfo) {}
virtual void HandleLoopingChanged() {}
private:
template <class S, typename R, typename... As>
auto ReturnTypeHelper(R (S::*)(As...)) -> R;
void Crash(const char* aReason, const char* aSite) {
char buf[1024];
SprintfLiteral(buf, "%s state=%s callsite=%s", aReason,
ToStateStr(GetState()), aSite);
MOZ_ReportAssertionFailure(buf, __FILE__, __LINE__);
MOZ_CRASH();
}
protected:
enum class EventVisibility : int8_t { Observable, Suppressed };
using Master = MediaDecoderStateMachine;
explicit StateObject(Master* aPtr) : mMaster(aPtr) {}
TaskQueue* OwnerThread() const { return mMaster->mTaskQueue; }
ReaderProxy* Reader() const { return mMaster->mReader; }
const MediaInfo& Info() const { return mMaster->Info(); }
MediaQueue<AudioData>& AudioQueue() const { return mMaster->mAudioQueue; }
MediaQueue<VideoData>& VideoQueue() const { return mMaster->mVideoQueue; }
template <class S, typename... Args, size_t... Indexes>
auto CallEnterMemberFunction(S* aS, Tuple<Args...>& aTuple,
std::index_sequence<Indexes...>)
-> decltype(ReturnTypeHelper(&S::Enter)) {
AUTO_PROFILER_LABEL("StateObject::CallEnterMemberFunction", MEDIA_PLAYBACK);
return aS->Enter(std::move(Get<Indexes>(aTuple))...);
}
// Note this function will delete the current state object.
// Don't access members to avoid UAF after this call.
template <class S, typename... Ts>
auto SetState(Ts&&... aArgs) -> decltype(ReturnTypeHelper(&S::Enter)) {
// |aArgs| must be passed by reference to avoid passing MOZ_NON_PARAM class
// SeekJob by value. See bug 1287006 and bug 1338374. But we still *must*
// copy the parameters, because |Exit()| can modify them. See bug 1312321.
// So we 1) pass the parameters by reference, but then 2) immediately copy
// them into a Tuple to be safe against modification, and finally 3) move
// the elements of the Tuple into the final function call.
auto copiedArgs = MakeTuple(std::forward<Ts>(aArgs)...);
// Copy mMaster which will reset to null.
auto master = mMaster;
auto* s = new S(master);
// It's possible to seek again during seeking, otherwise the new state
// should always be different from the original one.
MOZ_ASSERT(GetState() != s->GetState() ||
GetState() == DECODER_STATE_SEEKING_ACCURATE ||
GetState() == DECODER_STATE_SEEKING_FROMDORMANT ||
GetState() == DECODER_STATE_SEEKING_NEXTFRAMESEEKING ||
GetState() == DECODER_STATE_SEEKING_VIDEOONLY);
SLOG("change state to: %s", ToStateStr(s->GetState()));
Exit();
// Delete the old state asynchronously to avoid UAF if the caller tries to
// access its members after SetState() returns.
master->OwnerThread()->DispatchDirectTask(
NS_NewRunnableFunction("MDSM::StateObject::DeleteOldState",
[toDelete = std::move(master->mStateObj)]() {}));
// Also reset mMaster to catch potentail UAF.
mMaster = nullptr;
master->mStateObj.reset(s);
return CallEnterMemberFunction(s, copiedArgs,
std::index_sequence_for<Ts...>{});
}
RefPtr<MediaDecoder::SeekPromise> SetSeekingState(
SeekJob&& aSeekJob, EventVisibility aVisibility);
void SetDecodingState();
// Take a raw pointer in order not to change the life cycle of MDSM.
// It is guaranteed to be valid by MDSM.
Master* mMaster;
};
/**
* Purpose: decode metadata like duration and dimensions of the media resource.
*
* Transition to other states when decoding metadata is done:
* SHUTDOWN if failing to decode metadata.
* DECODING_FIRSTFRAME otherwise.
*/
class MediaDecoderStateMachine::DecodeMetadataState
: public MediaDecoderStateMachine::StateObject {
public:
explicit DecodeMetadataState(Master* aPtr) : StateObject(aPtr) {}
void Enter() {
MOZ_ASSERT(!mMaster->mVideoDecodeSuspended);
MOZ_ASSERT(!mMetadataRequest.Exists());
SLOG("Dispatching AsyncReadMetadata");
// We disconnect mMetadataRequest in Exit() so it is fine to capture
// a raw pointer here.
Reader()
->ReadMetadata()
->Then(
OwnerThread(), __func__,
[this](MetadataHolder&& aMetadata) {
OnMetadataRead(std::move(aMetadata));
},
[this](const MediaResult& aError) { OnMetadataNotRead(aError); })
->Track(mMetadataRequest);
}
void Exit() override { mMetadataRequest.DisconnectIfExists(); }
State GetState() const override { return DECODER_STATE_DECODING_METADATA; }
RefPtr<MediaDecoder::SeekPromise> HandleSeek(SeekTarget aTarget) override {
MOZ_DIAGNOSTIC_ASSERT(false, "Can't seek while decoding metadata.");
return MediaDecoder::SeekPromise::CreateAndReject(true, __func__);
}
void HandleVideoSuspendTimeout() override {
// Do nothing since no decoders are created yet.
}
void HandleResumeVideoDecoding(const TimeUnit&) override {
// We never suspend video decoding in this state.
MOZ_ASSERT(false, "Shouldn't have suspended video decoding.");
}
private:
void OnMetadataRead(MetadataHolder&& aMetadata);
void OnMetadataNotRead(const MediaResult& aError) {
AUTO_PROFILER_LABEL("DecodeMetadataState::OnMetadataNotRead",
MEDIA_PLAYBACK);
mMetadataRequest.Complete();
SLOGE("Decode metadata failed, shutting down decoder");
mMaster->DecodeError(aError);
}
MozPromiseRequestHolder<MediaFormatReader::MetadataPromise> mMetadataRequest;
};
/**
* Purpose: release decoder resources to save memory and hardware resources.
*
* Transition to:
* SEEKING if any seek request or play state changes to PLAYING.
*/
class MediaDecoderStateMachine::DormantState
: public MediaDecoderStateMachine::StateObject {
public:
explicit DormantState(Master* aPtr) : StateObject(aPtr) {}
void Enter() {
PROFILER_ADD_MARKER("MDSM::EnterDormantState", MEDIA_PLAYBACK);
if (mMaster->IsPlaying()) {
mMaster->StopPlayback();
}
// Calculate the position to seek to when exiting dormant.
auto t = mMaster->mMediaSink->IsStarted() ? mMaster->GetClock()
: mMaster->GetMediaTime();
mMaster->AdjustByLooping(t);
mPendingSeek.mTarget.emplace(t, SeekTarget::Accurate);
// SeekJob asserts |mTarget.IsValid() == !mPromise.IsEmpty()| so we
// need to create the promise even it is not used at all.
// The promise may be used when coming out of DormantState into
// SeekingState.
RefPtr<MediaDecoder::SeekPromise> x =
mPendingSeek.mPromise.Ensure(__func__);
// No need to call ResetDecode() and StopMediaSink() here.
// We will do them during seeking when exiting dormant.
// Ignore WAIT_FOR_DATA since we won't decode in dormant.
mMaster->mAudioWaitRequest.DisconnectIfExists();
mMaster->mVideoWaitRequest.DisconnectIfExists();
MaybeReleaseResources();
}
void Exit() override {
// mPendingSeek is either moved when exiting dormant or
// should be rejected here before transition to SHUTDOWN.
mPendingSeek.RejectIfExists(__func__);
}
State GetState() const override { return DECODER_STATE_DORMANT; }
RefPtr<MediaDecoder::SeekPromise> HandleSeek(SeekTarget aTarget) override;
void HandleVideoSuspendTimeout() override {
// Do nothing since we've released decoders in Enter().
}
void HandleResumeVideoDecoding(const TimeUnit&) override {
// Do nothing since we won't resume decoding until exiting dormant.
}
void HandlePlayStateChanged(MediaDecoder::PlayState aPlayState) override;
void HandleAudioDecoded(AudioData*) override { MaybeReleaseResources(); }
void HandleVideoDecoded(VideoData*, TimeStamp) override {
MaybeReleaseResources();
}
void HandleWaitingForAudio() override { MaybeReleaseResources(); }
void HandleWaitingForVideo() override { MaybeReleaseResources(); }
void HandleAudioCanceled() override { MaybeReleaseResources(); }
void HandleVideoCanceled() override { MaybeReleaseResources(); }
void HandleEndOfAudio() override { MaybeReleaseResources(); }
void HandleEndOfVideo() override { MaybeReleaseResources(); }
private:
void MaybeReleaseResources() {
if (!mMaster->mAudioDataRequest.Exists() &&
!mMaster->mVideoDataRequest.Exists()) {
// Release decoders only when they are idle. Otherwise it might cause
// decode error later when resetting decoders during seeking.
mMaster->mReader->ReleaseResources();
}
}
SeekJob mPendingSeek;
};
/**
* Purpose: decode the 1st audio and video frames to fire the 'loadeddata'
* event.
*
* Transition to:
* SHUTDOWN if any decode error.
* SEEKING if any seek request.
* DECODING/LOOPING_DECODING when the 'loadeddata' event is fired.
*/
class MediaDecoderStateMachine::DecodingFirstFrameState
: public MediaDecoderStateMachine::StateObject {
public:
explicit DecodingFirstFrameState(Master* aPtr) : StateObject(aPtr) {}
void Enter();
void Exit() override {
// mPendingSeek is either moved in MaybeFinishDecodeFirstFrame()
// or should be rejected here before transition to SHUTDOWN.
mPendingSeek.RejectIfExists(__func__);
}
State GetState() const override { return DECODER_STATE_DECODING_FIRSTFRAME; }
void HandleAudioDecoded(AudioData* aAudio) override {
mMaster->PushAudio(aAudio);
MaybeFinishDecodeFirstFrame();
}
void HandleVideoDecoded(VideoData* aVideo, TimeStamp aDecodeStart) override {
mMaster->PushVideo(aVideo);
MaybeFinishDecodeFirstFrame();
}
void HandleWaitingForAudio() override {
mMaster->WaitForData(MediaData::Type::AUDIO_DATA);
}
void HandleAudioCanceled() override { mMaster->RequestAudioData(); }
void HandleEndOfAudio() override {
AudioQueue().Finish();
MaybeFinishDecodeFirstFrame();
}
void HandleWaitingForVideo() override {
mMaster->WaitForData(MediaData::Type::VIDEO_DATA);
}
void HandleVideoCanceled() override {
mMaster->RequestVideoData(media::TimeUnit());
}
void HandleEndOfVideo() override {
VideoQueue().Finish();
MaybeFinishDecodeFirstFrame();
}
void HandleAudioWaited(MediaData::Type aType) override {
mMaster->RequestAudioData();
}
void HandleVideoWaited(MediaData::Type aType) override {
mMaster->RequestVideoData(media::TimeUnit());
}
void HandleVideoSuspendTimeout() override {
// Do nothing for we need to decode the 1st video frame to get the
// dimensions.
}
void HandleResumeVideoDecoding(const TimeUnit&) override {
// We never suspend video decoding in this state.
MOZ_ASSERT(false, "Shouldn't have suspended video decoding.");
}
RefPtr<MediaDecoder::SeekPromise> HandleSeek(SeekTarget aTarget) override {
if (mMaster->mIsMSE) {
return StateObject::HandleSeek(aTarget);
}
// Delay seek request until decoding first frames for non-MSE media.
SLOG("Not Enough Data to seek at this stage, queuing seek");
mPendingSeek.RejectIfExists(__func__);
mPendingSeek.mTarget.emplace(aTarget);
return mPendingSeek.mPromise.Ensure(__func__);
}
private:
// Notify FirstFrameLoaded if having decoded first frames and
// transition to SEEKING if there is any pending seek, or DECODING otherwise.
void MaybeFinishDecodeFirstFrame();
SeekJob mPendingSeek;
};
/**
* Purpose: decode audio/video data for playback.
*
* Transition to:
* DORMANT if playback is paused for a while.
* SEEKING if any seek request.
* SHUTDOWN if any decode error.
* BUFFERING if playback can't continue due to lack of decoded data.
* COMPLETED when having decoded all audio/video data.
* LOOPING_DECODING when media start seamless looping
*/
class MediaDecoderStateMachine::DecodingState
: public MediaDecoderStateMachine::StateObject {
public:
explicit DecodingState(Master* aPtr)
: StateObject(aPtr), mDormantTimer(OwnerThread()) {}
void Enter();
void Exit() override {
if (!mDecodeStartTime.IsNull()) {
TimeDuration decodeDuration = TimeStamp::Now() - mDecodeStartTime;
SLOG("Exiting DECODING, decoded for %.3lfs", decodeDuration.ToSeconds());
}
mDormantTimer.Reset();
mOnAudioPopped.DisconnectIfExists();
mOnVideoPopped.DisconnectIfExists();
}
void Step() override;
State GetState() const override { return DECODER_STATE_DECODING; }
void HandleAudioDecoded(AudioData* aAudio) override {
mMaster->PushAudio(aAudio);
DispatchDecodeTasksIfNeeded();
MaybeStopPrerolling();
}
void HandleVideoDecoded(VideoData* aVideo, TimeStamp aDecodeStart) override {
mMaster->PushVideo(aVideo);
DispatchDecodeTasksIfNeeded();
MaybeStopPrerolling();
}
void HandleAudioCanceled() override { mMaster->RequestAudioData(); }
void HandleVideoCanceled() override {
mMaster->RequestVideoData(mMaster->GetMediaTime());
}
void HandleEndOfAudio() override;
void HandleEndOfVideo() override;
void HandleWaitingForAudio() override {
mMaster->WaitForData(MediaData::Type::AUDIO_DATA);
MaybeStopPrerolling();
}
void HandleWaitingForVideo() override {
mMaster->WaitForData(MediaData::Type::VIDEO_DATA);
MaybeStopPrerolling();
}
void HandleAudioWaited(MediaData::Type aType) override {
mMaster->RequestAudioData();
}
void HandleVideoWaited(MediaData::Type aType) override {
mMaster->RequestVideoData(mMaster->GetMediaTime());
}
void HandleAudioCaptured() override {
MaybeStopPrerolling();
// MediaSink is changed. Schedule Step() to check if we can start playback.
mMaster->ScheduleStateMachine();
}
void HandleVideoSuspendTimeout() override {
// No video, so nothing to suspend.
if (!mMaster->HasVideo()) {
return;
}
mMaster->mVideoDecodeSuspended = true;
mMaster->mOnPlaybackEvent.Notify(MediaPlaybackEvent::EnterVideoSuspend);
Reader()->SetVideoBlankDecode(true);
}
void HandlePlayStateChanged(MediaDecoder::PlayState aPlayState) override {
if (aPlayState == MediaDecoder::PLAY_STATE_PLAYING) {
// Schedule Step() to check if we can start playback.
mMaster->ScheduleStateMachine();
// Try to dispatch decoding tasks for mMinimizePreroll might be reset.
DispatchDecodeTasksIfNeeded();
}
if (aPlayState == MediaDecoder::PLAY_STATE_PAUSED) {
StartDormantTimer();
} else {
mDormantTimer.Reset();
}
}
void GetDebugInfo(
dom::MediaDecoderStateMachineDecodingStateDebugInfo& aInfo) override {
aInfo.mIsPrerolling = mIsPrerolling;
}
void HandleLoopingChanged() override { SetDecodingState(); }
protected:
virtual void EnsureAudioDecodeTaskQueued();
private:
void DispatchDecodeTasksIfNeeded();
void EnsureVideoDecodeTaskQueued();
void MaybeStartBuffering();
// At the start of decoding we want to "preroll" the decode until we've
// got a few frames decoded before we consider whether decode is falling
// behind. Otherwise our "we're falling behind" logic will trigger
// unnecessarily if we start playing as soon as the first sample is
// decoded. These two fields store how many video frames and audio
// samples we must consume before are considered to be finished prerolling.
TimeUnit AudioPrerollThreshold() const {
return (mMaster->mAmpleAudioThreshold / 2)
.MultDouble(mMaster->mPlaybackRate);
}
uint32_t VideoPrerollFrames() const {
return std::min<uint32_t>(
(mMaster->GetAmpleVideoFrames() / 2) * mMaster->mPlaybackRate + 1,
sVideoQueueDefaultSize);
}
bool DonePrerollingAudio() {
return !mMaster->IsAudioDecoding() ||
mMaster->GetDecodedAudioDuration() >= AudioPrerollThreshold();
}
bool DonePrerollingVideo() {
return !mMaster->IsVideoDecoding() ||
static_cast<uint32_t>(mMaster->VideoQueue().GetSize()) >=
VideoPrerollFrames();
}
void MaybeStopPrerolling() {
if (mIsPrerolling &&
(DonePrerollingAudio() || mMaster->IsWaitingAudioData()) &&
(DonePrerollingVideo() || mMaster->IsWaitingVideoData())) {
mIsPrerolling = false;
// Check if we can start playback.
mMaster->ScheduleStateMachine();
}
}
void StartDormantTimer() {
if (!mMaster->mMediaSeekable) {
// Don't enter dormant if the media is not seekable because we need to
// seek when exiting dormant.
return;
}
auto timeout = StaticPrefs::media_dormant_on_pause_timeout_ms();
if (timeout < 0) {
// Disabled when timeout is negative.
return;
} else if (timeout == 0) {
// Enter dormant immediately without scheduling a timer.
SetState<DormantState>();
return;
}
if (mMaster->mMinimizePreroll) {
SetState<DormantState>();
return;
}
TimeStamp target =
TimeStamp::Now() + TimeDuration::FromMilliseconds(timeout);
mDormantTimer.Ensure(
target,
[this]() {
AUTO_PROFILER_LABEL("DecodingState::StartDormantTimer:SetDormant",
MEDIA_PLAYBACK);
mDormantTimer.CompleteRequest();
SetState<DormantState>();
},
[this]() { mDormantTimer.CompleteRequest(); });
}
// Time at which we started decoding.
TimeStamp mDecodeStartTime;
// When we start decoding (either for the first time, or after a pause)
// we may be low on decoded data. We don't want our "low data" logic to
// kick in and decide that we're low on decoded data because the download
// can't keep up with the decode, and cause us to pause playback. So we
// have a "preroll" stage, where we ignore the results of our "low data"
// logic during the first few frames of our decode. This occurs during
// playback.
bool mIsPrerolling = true;
// Fired when playback is paused for a while to enter dormant.
DelayedScheduler mDormantTimer;
MediaEventListener mOnAudioPopped;
MediaEventListener mOnVideoPopped;
};
/**
* Purpose: decode audio/video data for playback when media is in seamless
* looping, we will adjust media time to make samples time monotonically
* increasing.
*
* Transition to:
* DORMANT if playback is paused for a while.
* SEEKING if any seek request.
* SHUTDOWN if any decode error.
* BUFFERING if playback can't continue due to lack of decoded data.
* COMPLETED when having decoded all audio/video data.
* DECODING when media stop seamless looping
*/
class MediaDecoderStateMachine::LoopingDecodingState
: public MediaDecoderStateMachine::DecodingState {
public:
explicit LoopingDecodingState(Master* aPtr)
: DecodingState(aPtr), mIsReachingAudioEOS(!mMaster->IsAudioDecoding()) {
MOZ_ASSERT(mMaster->mLooping);
}
void Enter() {
if (mIsReachingAudioEOS) {
SLOG("audio has ended, request the data again.");
UpdatePlaybackPositionToZeroIfNeeded();
RequestAudioDataFromStartPosition();
}
DecodingState::Enter();
}
void Exit() override {
if (ShouldDiscardLoopedAudioData()) {
mMaster->mAudioDataRequest.DisconnectIfExists();
DiscardLoopedAudioData();
}
if (HasDecodedLastAudioFrame()) {
AudioQueue().Finish();
}
mAudioDataRequest.DisconnectIfExists();
mAudioSeekRequest.DisconnectIfExists();
DecodingState::Exit();
}
State GetState() const override { return DECODER_STATE_LOOPING_DECODING; }
void HandleAudioDecoded(AudioData* aAudio) override {
MediaResult rv = LoopingAudioTimeAdjustment(aAudio);
if (NS_WARN_IF(NS_FAILED(rv))) {
mMaster->DecodeError(rv);
return;
}
mMaster->mDecodedAudioEndTime =
std::max(aAudio->GetEndTime(), mMaster->mDecodedAudioEndTime);
SLOG("sample after time-adjustment [%" PRId64 ",%" PRId64 "]",
aAudio->mTime.ToMicroseconds(), aAudio->GetEndTime().ToMicroseconds());
DecodingState::HandleAudioDecoded(aAudio);
}
void HandleEndOfAudio() override {
mIsReachingAudioEOS = true;
// The data time in the audio queue is assumed to be increased linearly,
// so we need to add the last ending time as the offset to correct the
// audio data time in the next round when seamless looping is enabled.
mAudioLoopingOffset = mMaster->mDecodedAudioEndTime;
if (mMaster->mAudioDecodedDuration.isNothing()) {
mMaster->mAudioDecodedDuration.emplace(mMaster->mDecodedAudioEndTime);
}
SLOG(
"received EOS when seamless looping, starts seeking, "
"AudioLoopingOffset=[%" PRId64 "]",
mAudioLoopingOffset.ToMicroseconds());
RequestAudioDataFromStartPosition();
}
private:
void RequestAudioDataFromStartPosition() {
Reader()->ResetDecode(TrackInfo::kAudioTrack);
Reader()
->Seek(SeekTarget(media::TimeUnit::Zero(), SeekTarget::Accurate))
->Then(
OwnerThread(), __func__,
[this]() -> void {
AUTO_PROFILER_LABEL(
"LoopingDecodingState::RequestAudioDataFromStartPosition:"
"SeekResolved",
MEDIA_PLAYBACK);
mAudioSeekRequest.Complete();
SLOG(
"seeking completed, start to request first sample, "
"queueing audio task - queued=%zu, decoder-queued=%zu",
AudioQueue().GetSize(), Reader()->SizeOfAudioQueueInFrames());
Reader()
->RequestAudioData()
->Then(
OwnerThread(), __func__,
[this](RefPtr<AudioData> aAudio) {
AUTO_PROFILER_LABEL(
"LoopingDecodingState::"
"RequestAudioDataFromStartPosition:"
"RequestDataResolved",
MEDIA_PLAYBACK);
mIsReachingAudioEOS = false;
mAudioDataRequest.Complete();
SLOG(
"got audio decoded sample "
"[%" PRId64 ",%" PRId64 "]",
aAudio->mTime.ToMicroseconds(),
aAudio->GetEndTime().ToMicroseconds());
HandleAudioDecoded(aAudio);
},
[this](const MediaResult& aError) {
AUTO_PROFILER_LABEL(
"LoopingDecodingState::"
"RequestAudioDataFromStartPosition:"
"RequestDataRejected",
MEDIA_PLAYBACK);
mAudioDataRequest.Complete();
HandleError(aError);
})
->Track(mAudioDataRequest);
},
[this](const SeekRejectValue& aReject) -> void {
AUTO_PROFILER_LABEL(
"LoopingDecodingState::RequestAudioDataFromStartPosition:"
"SeekRejected",
MEDIA_PLAYBACK);
mAudioSeekRequest.Complete();
HandleError(aReject.mError);
})
->Track(mAudioSeekRequest);
}
void UpdatePlaybackPositionToZeroIfNeeded() {
MOZ_ASSERT(mIsReachingAudioEOS);
MOZ_ASSERT(mAudioLoopingOffset == media::TimeUnit::Zero());
// If we have already reached EOS before starting media sink, the sink
// has not started yet and the current position is larger than last decoded
// end time, that means we directly seeked to EOS and playback would start
// from the start position soon. Therefore, we should reset the position to
// 0s so that when media sink starts we can make it start from 0s, not from
// EOS position which would result in wrong estimation of decoded audio
// duration because decoded data's time which can't be adjusted as offset is
// zero would be always less than media sink time.
if (!mMaster->mMediaSink->IsStarted() &&
mMaster->mCurrentPosition.Ref() > mMaster->mDecodedAudioEndTime) {
mMaster->UpdatePlaybackPositionInternal(TimeUnit::Zero());
}
}
void HandleError(const MediaResult& aError);
void EnsureAudioDecodeTaskQueued() override {
if (mAudioSeekRequest.Exists() || mAudioDataRequest.Exists()) {
return;
}
DecodingState::EnsureAudioDecodeTaskQueued();
}
MediaResult LoopingAudioTimeAdjustment(AudioData* aAudio) {
if (mAudioLoopingOffset != media::TimeUnit::Zero()) {
aAudio->mTime += mAudioLoopingOffset;
}
return aAudio->mTime.IsValid()
? MediaResult(NS_OK)
: MediaResult(
NS_ERROR_DOM_MEDIA_OVERFLOW_ERR,
"Audio sample overflow during looping time adjustment");
}
bool ShouldDiscardLoopedAudioData() const {
if (!mMaster->mMediaSink->IsStarted()) {
return false;
}
/**
* If media cancels looping, we should check whether there are audio data
* whose time is later than EOS. If so, we should discard them because we
* won't have a chance to play them.
*
* playback last decoded
* position EOS data time
* ----|---------------|------------|---------> (Increasing timeline)
* mCurrent mLooping mMaster's
* ClockTime Offset mDecodedAudioEndTime
*
*/
return (mAudioLoopingOffset != media::TimeUnit::Zero() &&
mMaster->GetClock() < mAudioLoopingOffset &&
mAudioLoopingOffset < mMaster->mDecodedAudioEndTime);
}
void DiscardLoopedAudioData() {
if (mAudioLoopingOffset == media::TimeUnit::Zero()) {
return;
}
SLOG("Discard frames after the time=%" PRId64,
mAudioLoopingOffset.ToMicroseconds());
DiscardFramesFromTail(AudioQueue(), [&](int64_t aSampleTime) {
return aSampleTime > mAudioLoopingOffset.ToMicroseconds();
});
}
bool HasDecodedLastAudioFrame() const {
// when we're going to leave looping state and have got EOS before, we
// should mark audio queue as ended because we have got all data we need.
return mAudioDataRequest.Exists() || mAudioSeekRequest.Exists() ||
ShouldDiscardLoopedAudioData();
}
bool mIsReachingAudioEOS;
media::TimeUnit mAudioLoopingOffset = media::TimeUnit::Zero();
MozPromiseRequestHolder<MediaFormatReader::SeekPromise> mAudioSeekRequest;
MozPromiseRequestHolder<AudioDataPromise> mAudioDataRequest;
};
/**
* Purpose: seek to a particular new playback position.
*
* Transition to:
* SEEKING if any new seek request.
* SHUTDOWN if seek failed.
* COMPLETED if the new playback position is the end of the media resource.
* NextFrameSeekingState if completing a NextFrameSeekingFromDormantState.
* DECODING/LOOPING_DECODING otherwise.
*/
class MediaDecoderStateMachine::SeekingState
: public MediaDecoderStateMachine::StateObject {
public:
explicit SeekingState(Master* aPtr)
: StateObject(aPtr), mVisibility(static_cast<EventVisibility>(0)) {}
RefPtr<MediaDecoder::SeekPromise> Enter(SeekJob&& aSeekJob,
EventVisibility aVisibility) {
mSeekJob = std::move(aSeekJob);
mVisibility = aVisibility;
// Suppressed visibility comes from two cases: (1) leaving dormant state,
// and (2) resuming suspended video decoder. We want both cases to be
// transparent to the user. So we only notify the change when the seek
// request is from the user.
if (mVisibility == EventVisibility::Observable) {
// Don't stop playback for a video-only seek since we want to keep playing
// audio and we don't need to stop playback while leaving dormant for the
// playback should has been stopped.
mMaster->StopPlayback();
mMaster->UpdatePlaybackPositionInternal(mSeekJob.mTarget->GetTime());
mMaster->mOnPlaybackEvent.Notify(MediaPlaybackEvent::SeekStarted);
mMaster->mOnNextFrameStatus.Notify(
MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_SEEKING);
}
RefPtr<MediaDecoder::SeekPromise> p = mSeekJob.mPromise.Ensure(__func__);
DoSeek();
return p;
}
virtual void Exit() override = 0;
State GetState() const override = 0;
void HandleAudioDecoded(AudioData* aAudio) override = 0;
void HandleVideoDecoded(VideoData* aVideo,
TimeStamp aDecodeStart) override = 0;
void HandleAudioWaited(MediaData::Type aType) override = 0;
void HandleVideoWaited(MediaData::Type aType) override = 0;
void HandleVideoSuspendTimeout() override {
// Do nothing since we want a valid video frame to show when seek is done.
}
void HandleResumeVideoDecoding(const TimeUnit&) override {
// Do nothing. We will resume video decoding in the decoding state.
}
// We specially handle next frame seeks by ignoring them if we're already
// seeking.
RefPtr<MediaDecoder::SeekPromise> HandleSeek(SeekTarget aTarget) override {
if (aTarget.IsNextFrame()) {
// We ignore next frame seeks if we already have a seek pending
SLOG("Already SEEKING, ignoring seekToNextFrame");
MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished");
return MediaDecoder::SeekPromise::CreateAndReject(/* aIgnored = */ true,
__func__);
}
return StateObject::HandleSeek(aTarget);
}
protected:
SeekJob mSeekJob;
EventVisibility mVisibility;
virtual void DoSeek() = 0;
// Transition to the next state (defined by the subclass) when seek is
// completed.
virtual void GoToNextState() { SetDecodingState(); }
void SeekCompleted();
virtual TimeUnit CalculateNewCurrentTime() const = 0;
};
class MediaDecoderStateMachine::AccurateSeekingState
: public MediaDecoderStateMachine::SeekingState {
public:
explicit AccurateSeekingState(Master* aPtr) : SeekingState(aPtr) {}
State GetState() const override { return DECODER_STATE_SEEKING_ACCURATE; }
RefPtr<MediaDecoder::SeekPromise> Enter(SeekJob&& aSeekJob,
EventVisibility aVisibility) {
MOZ_ASSERT(aSeekJob.mTarget->IsAccurate() || aSeekJob.mTarget->IsFast());
mCurrentTimeBeforeSeek = mMaster->GetMediaTime();
return SeekingState::Enter(std::move(aSeekJob), aVisibility);
}
void Exit() override {
// Disconnect MediaDecoder.
mSeekJob.RejectIfExists(__func__);
// Disconnect ReaderProxy.
mSeekRequest.DisconnectIfExists();
mWaitRequest.DisconnectIfExists();
}
void HandleAudioDecoded(AudioData* aAudio) override {
MOZ_ASSERT(!mDoneAudioSeeking || !mDoneVideoSeeking,
"Seek shouldn't be finished");
MOZ_ASSERT(aAudio);
AdjustFastSeekIfNeeded(aAudio);
if (mSeekJob.mTarget->IsFast()) {
// Non-precise seek; we can stop the seek at the first sample.
mMaster->PushAudio(aAudio);
mDoneAudioSeeking = true;
} else {
nsresult rv = DropAudioUpToSeekTarget(aAudio);
if (NS_FAILED(rv)) {
mMaster->DecodeError(rv);
return;
}
}
if (!mDoneAudioSeeking) {
RequestAudioData();
return;
}
MaybeFinishSeek();
}
void HandleVideoDecoded(VideoData* aVideo, TimeStamp aDecodeStart) override {
MOZ_ASSERT(!mDoneAudioSeeking || !mDoneVideoSeeking,
"Seek shouldn't be finished");
MOZ_ASSERT(aVideo);
AdjustFastSeekIfNeeded(aVideo);
if (mSeekJob.mTarget->IsFast()) {
// Non-precise seek. We can stop the seek at the first sample.
mMaster->PushVideo(aVideo);
mDoneVideoSeeking = true;
} else {
nsresult rv = DropVideoUpToSeekTarget(aVideo);
if (NS_FAILED(rv)) {
mMaster->DecodeError(rv);
return;
}
}
if (!mDoneVideoSeeking) {
RequestVideoData();
return;
}
MaybeFinishSeek();
}
void HandleWaitingForAudio() override {
MOZ_ASSERT(!mDoneAudioSeeking);
mMaster->WaitForData(MediaData::Type::AUDIO_DATA);
}
void HandleAudioCanceled() override {
MOZ_ASSERT(!mDoneAudioSeeking);
RequestAudioData();
}
void HandleEndOfAudio() override {
HandleEndOfAudioInternal();
MaybeFinishSeek();
}
void HandleWaitingForVideo() override {
MOZ_ASSERT(!mDoneVideoSeeking);
mMaster->WaitForData(MediaData::Type::VIDEO_DATA);
}
void HandleVideoCanceled() override {
MOZ_ASSERT(!mDoneVideoSeeking);
RequestVideoData();
}
void HandleEndOfVideo() override {
HandleEndOfVideoInternal();
MaybeFinishSeek();
}
void HandleAudioWaited(MediaData::Type aType) override {
MOZ_ASSERT(!mDoneAudioSeeking || !mDoneVideoSeeking,
"Seek shouldn't be finished");
RequestAudioData();
}
void HandleVideoWaited(MediaData::Type aType) override {
MOZ_ASSERT(!mDoneAudioSeeking || !mDoneVideoSeeking,
"Seek shouldn't be finished");
RequestVideoData();
}
void DoSeek() override {
mDoneAudioSeeking = !Info().HasAudio();
mDoneVideoSeeking = !Info().HasVideo();
mMaster->ResetDecode();
mMaster->StopMediaSink();
DemuxerSeek();
}
TimeUnit CalculateNewCurrentTime() const override {
const auto seekTime = mSeekJob.mTarget->GetTime();
// For the accurate seek, we always set the newCurrentTime = seekTime so
// that the updated HTMLMediaElement.currentTime will always be the seek
// target; we rely on the MediaSink to handles the gap between the
// newCurrentTime and the real decoded samples' start time.
if (mSeekJob.mTarget->IsAccurate()) {
return seekTime;
}
// For the fast seek, we update the newCurrentTime with the decoded audio
// and video samples, set it to be the one which is closet to the seekTime.
if (mSeekJob.mTarget->IsFast()) {
RefPtr<AudioData> audio = AudioQueue().PeekFront();
RefPtr<VideoData> video = VideoQueue().PeekFront();
// A situation that both audio and video approaches the end.
if (!audio && !video) {
return seekTime;
}
const int64_t audioStart =
audio ? audio->mTime.ToMicroseconds() : INT64_MAX;
const int64_t videoStart =
video ? video->mTime.ToMicroseconds() : INT64_MAX;
const int64_t audioGap = std::abs(audioStart - seekTime.ToMicroseconds());
const int64_t videoGap = std::abs(videoStart - seekTime.ToMicroseconds());
return TimeUnit::FromMicroseconds(audioGap <= videoGap ? audioStart
: videoStart);
}
MOZ_ASSERT(false, "AccurateSeekTask doesn't handle other seek types.");
return TimeUnit::Zero();
}
protected:
void DemuxerSeek() {
// Request the demuxer to perform seek.
Reader()
->Seek(mSeekJob.mTarget.ref())
->Then(
OwnerThread(), __func__,
[this](const media::TimeUnit& aUnit) { OnSeekResolved(aUnit); },
[this](const SeekRejectValue& aReject) { OnSeekRejected(aReject); })
->Track(mSeekRequest);
}
void OnSeekResolved(media::TimeUnit) {
AUTO_PROFILER_LABEL("AccurateSeekingState::OnSeekResolved", MEDIA_PLAYBACK);
mSeekRequest.Complete();
// We must decode the first samples of active streams, so we can determine
// the new stream time. So dispatch tasks to do that.
if (!mDoneVideoSeeking) {
RequestVideoData();
}
if (!mDoneAudioSeeking) {
RequestAudioData();
}
}
void OnSeekRejected(const SeekRejectValue& aReject) {
AUTO_PROFILER_LABEL("AccurateSeekingState::OnSeekRejected", MEDIA_PLAYBACK);
mSeekRequest.Complete();
if (aReject.mError == NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA) {
SLOG("OnSeekRejected reason=WAITING_FOR_DATA type=%s",
MediaData::TypeToStr(aReject.mType));
MOZ_ASSERT_IF(aReject.mType == MediaData::Type::AUDIO_DATA,
!mMaster->IsRequestingAudioData());
MOZ_ASSERT_IF(aReject.mType == MediaData::Type::VIDEO_DATA,
!mMaster->IsRequestingVideoData());
MOZ_ASSERT_IF(aReject.mType == MediaData::Type::AUDIO_DATA,
!mMaster->IsWaitingAudioData());
MOZ_ASSERT_IF(aReject.mType == MediaData::Type::VIDEO_DATA,
!mMaster->IsWaitingVideoData());
// Fire 'waiting' to notify the player that we are waiting for data.
mMaster->mOnNextFrameStatus.Notify(
MediaDecoderOwner::NEXT_FRAME_UNAVAILABLE_SEEKING);
Reader()
->WaitForData(aReject.mType)
->Then(
OwnerThread(), __func__,
[this](MediaData::Type aType) {
AUTO_PROFILER_LABEL(
"AccurateSeekingState::OnSeekRejected:WaitDataResolved",
MEDIA_PLAYBACK);
SLOG("OnSeekRejected wait promise resolved");
mWaitRequest.Complete();
DemuxerSeek();
},
[this](const WaitForDataRejectValue& aRejection) {
AUTO_PROFILER_LABEL(
"AccurateSeekingState::OnSeekRejected:WaitDataRejected",
MEDIA_PLAYBACK);
SLOG("OnSeekRejected wait promise rejected");
mWaitRequest.Complete();
mMaster->DecodeError(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA);
})
->Track(mWaitRequest);
return;
}
if (aReject.mError == NS_ERROR_DOM_MEDIA_END_OF_STREAM) {
if (!mDoneAudioSeeking) {
HandleEndOfAudioInternal();
}
if (!mDoneVideoSeeking) {
HandleEndOfVideoInternal();
}
MaybeFinishSeek();
return;
}
MOZ_ASSERT(NS_FAILED(aReject.mError),
"Cancels should also disconnect mSeekRequest");
mMaster->DecodeError(aReject.mError);
}
void RequestAudioData() {
MOZ_ASSERT(!mDoneAudioSeeking);
mMaster->RequestAudioData();
}
virtual void RequestVideoData() {
MOZ_ASSERT(!mDoneVideoSeeking);
mMaster->RequestVideoData(media::TimeUnit());
}
void AdjustFastSeekIfNeeded(MediaData* aSample) {
if (mSeekJob.mTarget->IsFast() &&
mSeekJob.mTarget->GetTime() > mCurrentTimeBeforeSeek &&
aSample->mTime < mCurrentTimeBeforeSeek) {
// We are doing a fastSeek, but we ended up *before* the previous
// playback position. This is surprising UX, so switch to an accurate
// seek and decode to the seek target. This is not conformant to the
// spec, fastSeek should always be fast, but until we get the time to
// change all Readers to seek to the keyframe after the currentTime
// in this case, we'll just decode forward. Bug 1026330.
mSeekJob.mTarget->SetType(SeekTarget::Accurate);
}
}
nsresult DropAudioUpToSeekTarget(AudioData* aAudio) {
MOZ_ASSERT(aAudio && mSeekJob.mTarget->IsAccurate());
if (mSeekJob.mTarget->GetTime() >= aAudio->GetEndTime()) {
// Our seek target lies after the frames in this AudioData. Don't
// push it onto the audio queue, and keep decoding forwards.
return NS_OK;
}
if (aAudio->mTime > mSeekJob.mTarget->GetTime()) {
// The seek target doesn't lie in the audio block just after the last
// audio frames we've seen which were before the seek target. This
// could have been the first audio data we've seen after seek, i.e. the
// seek terminated after the seek target in the audio stream. Just
// abort the audio decode-to-target, the state machine will play
// silence to cover the gap. Typically this happens in poorly muxed
// files.
SLOGW("Audio not synced after seek, maybe a poorly muxed file?");
mMaster->PushAudio(aAudio);
mDoneAudioSeeking = true;
return NS_OK;
}
bool ok = aAudio->SetTrimWindow(
{mSeekJob.mTarget->GetTime(), aAudio->GetEndTime()});
if (!ok) {
return NS_ERROR_DOM_MEDIA_OVERFLOW_ERR;
}
MOZ_ASSERT(AudioQueue().GetSize() == 0,
"Should be the 1st sample after seeking");
mMaster->PushAudio(aAudio);
mDoneAudioSeeking = true;
return NS_OK;
}
nsresult DropVideoUpToSeekTarget(VideoData* aVideo) {
MOZ_ASSERT(aVideo);
SLOG("DropVideoUpToSeekTarget() frame [%" PRId64 ", %" PRId64 "]",
aVideo->mTime.ToMicroseconds(), aVideo->GetEndTime().ToMicroseconds());
const auto target = GetSeekTarget();
// If the frame end time is less than the seek target, we won't want
// to display this frame after the seek, so discard it.
if (target >= aVideo->GetEndTime()) {
SLOG("DropVideoUpToSeekTarget() pop video frame [%" PRId64 ", %" PRId64
"] target=%" PRId64,
aVideo->mTime.ToMicroseconds(),
aVideo->GetEndTime().ToMicroseconds(), target.ToMicroseconds());
PROFILER_ADD_MARKER("MDSM::DropVideoUpToSeekTarget", MEDIA_PLAYBACK);
mFirstVideoFrameAfterSeek = aVideo;
} else {
if (target >= aVideo->mTime && aVideo->GetEndTime() >= target) {
// The seek target lies inside this frame's time slice. Adjust the
// frame's start time to match the seek target.
aVideo->UpdateTimestamp(target);
}
mFirstVideoFrameAfterSeek = nullptr;
SLOG("DropVideoUpToSeekTarget() found video frame [%" PRId64 ", %" PRId64
"] containing target=%" PRId64,
aVideo->mTime.ToMicroseconds(),
aVideo->GetEndTime().ToMicroseconds(), target.ToMicroseconds());
MOZ_ASSERT(VideoQueue().GetSize() == 0,
"Should be the 1st sample after seeking");
mMaster->PushVideo(aVideo);
mDoneVideoSeeking = true;
}
return NS_OK;
}
void HandleEndOfAudioInternal() {
MOZ_ASSERT(!mDoneAudioSeeking);
AudioQueue().Finish();
mDoneAudioSeeking = true;
}
void HandleEndOfVideoInternal() {
MOZ_ASSERT(!mDoneVideoSeeking);
if (mFirstVideoFrameAfterSeek) {
// Hit the end of stream. Move mFirstVideoFrameAfterSeek into
// mSeekedVideoData so we have something to display after seeking.
mMaster->PushVideo(mFirstVideoFrameAfterSeek);
}
VideoQueue().Finish();
mDoneVideoSeeking = true;
}
void MaybeFinishSeek() {
if (mDoneAudioSeeking && mDoneVideoSeeking) {
SeekCompleted();
}
}
/*
* Track the current seek promise made by the reader.
*/
MozPromiseRequestHolder<MediaFormatReader::SeekPromise> mSeekRequest;
/*
* Internal state.
*/
media::TimeUnit mCurrentTimeBeforeSeek;
bool mDoneAudioSeeking = false;
bool mDoneVideoSeeking = false;
MozPromiseRequestHolder<WaitForDataPromise> mWaitRequest;
// This temporarily stores the first frame we decode after we seek.
// This is so that if we hit end of stream while we're decoding to reach
// the seek target, we will still have a frame that we can display as the
// last frame in the media.
RefPtr<VideoData> mFirstVideoFrameAfterSeek;
private:
virtual media::TimeUnit GetSeekTarget() const {
return mSeekJob.mTarget->GetTime();
}
};
/*
* Remove samples from the queue until aCompare() returns false.
* aCompare A function object with the signature bool(int64_t) which returns
* true for samples that should be removed.
*/
template <typename Type, typename Function>
static void DiscardFrames(MediaQueue<Type>& aQueue, const Function& aCompare) {
while (aQueue.GetSize() > 0) {
if (aCompare(aQueue.PeekFront()->mTime.ToMicroseconds())) {
RefPtr<Type> releaseMe = aQueue.PopFront();
continue;
}
break;
}
}
class MediaDecoderStateMachine::NextFrameSeekingState
: public MediaDecoderStateMachine::SeekingState {
public:
explicit NextFrameSeekingState(Master* aPtr) : SeekingState(aPtr) {}
State GetState() const override {
return DECODER_STATE_SEEKING_NEXTFRAMESEEKING;
}
RefPtr<MediaDecoder::SeekPromise> Enter(SeekJob&& aSeekJob,
EventVisibility aVisibility) {
MOZ_ASSERT(aSeekJob.mTarget->IsNextFrame());
mCurrentTime = mMaster->GetMediaTime();
mDuration = mMaster->Duration();
return SeekingState::Enter(std::move(aSeekJob), aVisibility);
}
void Exit() override {
// Disconnect my async seek operation.
if (mAsyncSeekTask) {
mAsyncSeekTask->Cancel();
}
// Disconnect MediaDecoder.
mSeekJob.RejectIfExists(__func__);
}
void HandleAudioDecoded(AudioData* aAudio) override {
mMaster->PushAudio(aAudio);
}
void HandleVideoDecoded(VideoData* aVideo, TimeStamp aDecodeStart) override {
MOZ_ASSERT(aVideo);
MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished");
MOZ_ASSERT(NeedMoreVideo());
if (aVideo->mTime > mCurrentTime) {
mMaster->PushVideo(aVideo);
FinishSeek();
} else {
RequestVideoData();
}
}
void HandleWaitingForAudio() override {
MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished");
// We don't care about audio decode errors in this state which will be
// handled by other states after seeking.
}
void HandleAudioCanceled() override {
MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished");
// We don't care about audio decode errors in this state which will be
// handled by other states after seeking.
}
void HandleEndOfAudio() override {
MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished");
// We don't care about audio decode errors in this state which will be
// handled by other states after seeking.
}
void HandleWaitingForVideo() override {
MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished");
MOZ_ASSERT(NeedMoreVideo());
mMaster->WaitForData(MediaData::Type::VIDEO_DATA);
}
void HandleVideoCanceled() override {
MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished");
MOZ_ASSERT(NeedMoreVideo());
RequestVideoData();
}
void HandleEndOfVideo() override {
MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished");
MOZ_ASSERT(NeedMoreVideo());
VideoQueue().Finish();
FinishSeek();
}
void HandleAudioWaited(MediaData::Type aType) override {
// We don't care about audio in this state.
}
void HandleVideoWaited(MediaData::Type aType) override {
MOZ_ASSERT(!mSeekJob.mPromise.IsEmpty(), "Seek shouldn't be finished");
MOZ_ASSERT(NeedMoreVideo());
RequestVideoData();
}
TimeUnit CalculateNewCurrentTime() const override {
// The HTMLMediaElement.currentTime should be updated to the seek target
// which has been updated to the next frame's time.
return mSeekJob.mTarget->GetTime();
}
void DoSeek() override {
auto currentTime = mCurrentTime;
DiscardFrames(VideoQueue(), [currentTime](int64_t aSampleTime) {
return aSampleTime <= currentTime.ToMicroseconds();
});
// If there is a pending video request, finish the seeking if we don't need
// more data, or wait for HandleVideoDecoded() to finish seeking.
if (mMaster->IsRequestingVideoData()) {
if (!NeedMoreVideo()) {
FinishSeek();
}
return;
}
// Otherwise, we need to do the seek operation asynchronously for a special
// case (bug504613.ogv) which has no data at all, the 1st seekToNextFrame()
// operation reaches the end of the media. If we did the seek operation
// synchronously, we immediately resolve the SeekPromise in mSeekJob and
// then switch to the CompletedState which dispatches an "ended" event.
// However, the ThenValue of the SeekPromise has not yet been set, so the
// promise resolving is postponed and then the JS developer receives the
// "ended" event before the seek promise is resolved.
// An asynchronous seek operation helps to solve this issue since while the
// seek is actually performed, the ThenValue of SeekPromise has already
// been set so that it won't be postponed.
RefPtr<Runnable> r = mAsyncSeekTask = new AysncNextFrameSeekTask(this);
nsresult rv = OwnerThread()->Dispatch(r.forget());
MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
Unused << rv;
}
private:
void DoSeekInternal() {
// We don't need to discard frames to the mCurrentTime here because we have
// done it at DoSeek() and any video data received in between either
// finishes the seek operation or be discarded, see HandleVideoDecoded().
if (!NeedMoreVideo()) {
FinishSeek();
} else if (!mMaster->IsRequestingVideoData() &&
!mMaster->IsWaitingVideoData()) {
RequestVideoData();
}
}
class AysncNextFrameSeekTask : public Runnable {
public:
explicit AysncNextFrameSeekTask(NextFrameSeekingState* aStateObject)
: Runnable(
"MediaDecoderStateMachine::NextFrameSeekingState::"
"AysncNextFrameSeekTask"),
mStateObj(aStateObject) {}
void Cancel() { mStateObj = nullptr; }
NS_IMETHOD Run() override {
if (mStateObj) {
AUTO_PROFILER_LABEL("AysncNextFrameSeekTask::Run", MEDIA_PLAYBACK);
mStateObj->DoSeekInternal();
}
return NS_OK;
}
private:
NextFrameSeekingState* mStateObj;
};
void RequestVideoData() { mMaster->RequestVideoData(media::TimeUnit()); }
bool NeedMoreVideo() const {
// Need to request video when we have none and video queue is not finished.
return VideoQueue().GetSize() == 0 && !VideoQueue().IsFinished();
}
// Update the seek target's time before resolving this seek task, the updated
// time will be used in the MDSM::SeekCompleted() to update the MDSM's
// position.
void UpdateSeekTargetTime() {
RefPtr<VideoData> data = VideoQueue().PeekFront();
if (data) {
mSeekJob.mTarget->SetTime(data->mTime);
} else {
MOZ_ASSERT(VideoQueue().AtEndOfStream());
mSeekJob.mTarget->SetTime(mDuration);
}
}
void FinishSeek() {
MOZ_ASSERT(!NeedMoreVideo());
UpdateSeekTargetTime();
auto time = mSeekJob.mTarget->GetTime().ToMicroseconds();
DiscardFrames(AudioQueue(),
[time](int64_t aSampleTime) { return aSampleTime < time; });
SeekCompleted();
}
/*
* Internal state.
*/
TimeUnit mCurrentTime;
TimeUnit mDuration;
RefPtr<AysncNextFrameSeekTask> mAsyncSeekTask;
};
class MediaDecoderStateMachine::NextFrameSeekingFromDormantState
: public MediaDecoderStateMachine::AccurateSeekingState {
public:
explicit NextFrameSeekingFromDormantState(Master* aPtr)
: AccurateSeekingState(aPtr) {}
State GetState() const override { return DECODER_STATE_SEEKING_FROMDORMANT; }
RefPtr<MediaDecoder::SeekPromise> Enter(SeekJob&& aCurrentSeekJob,
SeekJob&& aFutureSeekJob) {
mFutureSeekJob = std::move(aFutureSeekJob);
AccurateSeekingState::Enter(std::move(aCurrentSeekJob),
EventVisibility::Suppressed);
// Once seekToNextFrame() is called, we assume the user is likely to keep
// calling seekToNextFrame() repeatedly, and so, we should prevent the MDSM
// from getting into Dormant state.
mMaster->mMinimizePreroll = false;
return mFutureSeekJob.mPromise.Ensure(__func__);
}
void Exit() override {
mFutureSeekJob.RejectIfExists(__func__);
AccurateSeekingState::Exit();
}
private:
SeekJob mFutureSeekJob;
// We don't want to transition to DecodingState once this seek completes,
// instead, we transition to NextFrameSeekingState.
void GoToNextState() override {
SetState<NextFrameSeekingState>(std::move(mFutureSeekJob),
EventVisibility::Observable);
}
};
class MediaDecoderStateMachine::VideoOnlySeekingState
: public MediaDecoderStateMachine::AccurateSeekingState {
public:
explicit VideoOnlySeekingState(Master* aPtr) : AccurateSeekingState(aPtr) {}
State GetState() const override { return DECODER_STATE_SEEKING_VIDEOONLY; }
RefPtr<MediaDecoder::SeekPromise> Enter(SeekJob&& aSeekJob,
EventVisibility aVisibility) {
MOZ_ASSERT(aSeekJob.mTarget->IsVideoOnly());
MOZ_ASSERT(aVisibility == EventVisibility::Suppressed);
RefPtr<MediaDecoder::SeekPromise> p =
AccurateSeekingState::Enter(std::move(aSeekJob), aVisibility);
// Dispatch a mozvideoonlyseekbegin event to indicate UI for corresponding
// changes.
mMaster->mOnPlaybackEvent.Notify(MediaPlaybackEvent::VideoOnlySeekBegin);
return p;
}
void Exit() override {
// We are completing or discarding this video-only seek operation now,
// dispatch an event so that the UI can change in response to the end
// of video-only seek.
mMaster->mOnPlaybackEvent.Notify(
MediaPlaybackEvent::VideoOnlySeekCompleted);
AccurateSeekingState::Exit();
}
void HandleAudioDecoded(AudioData* aAudio) override {
MOZ_ASSERT(mDoneAudioSeeking && !mDoneVideoSeeking,
"Seek shouldn't be finished");
MOZ_ASSERT(aAudio);
// Video-only seek doesn't reset audio decoder. There might be pending audio
// requests when AccurateSeekTask::Seek() begins. We will just store the
// data without checking |mDiscontinuity| or calling
// DropAudioUpToSeekTarget().
mMaster->PushAudio(aAudio);
}
void HandleWaitingForAudio() override {}
void HandleAudioCanceled() override {}
void HandleEndOfAudio() override {}
void HandleAudioWaited(MediaData::Type aType) override {
MOZ_ASSERT(!mDoneAudioSeeking || !mDoneVideoSeeking,
"Seek shouldn't be finished");
// Ignore pending requests from video-only seek.
}
void DoSeek() override {
// TODO: keep decoding audio.
mDoneAudioSeeking = true;
mDoneVideoSeeking = !Info().HasVideo();
mMaster->ResetDecode(TrackInfo::kVideoTrack);
DemuxerSeek();
}
protected:
// Allow skip-to-next-key-frame to kick in if we fall behind the current
// playback position so decoding has a better chance to catch up.
void RequestVideoData() override {
MOZ_ASSERT(!mDoneVideoSeeking);
const auto& clock = mMaster->mMediaSink->IsStarted()
? mMaster->GetClock()
: mMaster->GetMediaTime();
const auto& nextKeyFrameTime = GetNextKeyFrameTime();
auto threshold = clock;
if (nextKeyFrameTime.IsValid() &&
clock >= (nextKeyFrameTime - sSkipToNextKeyFrameThreshold)) {
threshold = nextKeyFrameTime;
}
mMaster->RequestVideoData(threshold);
}
private:
// Trigger skip to next key frame if the current playback position is very
// close the next key frame's time.
static constexpr TimeUnit sSkipToNextKeyFrameThreshold =
TimeUnit::FromMicroseconds(5000);
// If the media is playing, drop video until catch up playback position.
media::TimeUnit GetSeekTarget() const override {
return mMaster->mMediaSink->IsStarted() ? mMaster->GetClock()
: mSeekJob.mTarget->GetTime();
}
media::TimeUnit GetNextKeyFrameTime() const {
// We only call this method in RequestVideoData() and we only request video
// data if we haven't done video seeking.
MOZ_DIAGNOSTIC_ASSERT(!mDoneVideoSeeking);
MOZ_DIAGNOSTIC_ASSERT(mMaster->VideoQueue().GetSize() == 0);
if (mFirstVideoFrameAfterSeek) {
return mFirstVideoFrameAfterSeek->NextKeyFrameTime();
}
return TimeUnit::Invalid();
}
};
constexpr TimeUnit MediaDecoderStateMachine::VideoOnlySeekingState::
sSkipToNextKeyFrameThreshold;
RefPtr<MediaDecoder::SeekPromise>
MediaDecoderStateMachine::DormantState::HandleSeek(SeekTarget aTarget) {
if (aTarget.IsNextFrame()) {
// NextFrameSeekingState doesn't reset the decoder unlike
// AccurateSeekingState. So we first must come out of dormant by seeking to
// mPendingSeek and continue later with the NextFrameSeek
SLOG("Changed state to SEEKING (to %" PRId64 ")",
aTarget.GetTime().ToMicroseconds());
SeekJob seekJob;
seekJob.mTarget = Some(aTarget);
return StateObject::SetState<NextFrameSeekingFromDormantState>(
std::move(mPendingSeek), std::move(seekJob));
}
return StateObject::HandleSeek(aTarget);