Source code

Revision control

Other Tools

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "TrackBuffersManager.h"
#include "ContainerParser.h"
#include "MediaSourceDemuxer.h"
#include "MediaSourceUtils.h"
#include "SourceBuffer.h"
#include "SourceBufferResource.h"
#include "SourceBufferTask.h"
#include "WebMDemuxer.h"
#include "mozilla/ErrorResult.h"
#include "mozilla/Preferences.h"
#include "mozilla/StaticPrefs_media.h"
#include "nsMimeTypes.h"
#ifdef MOZ_FMP4
# include "MP4Demuxer.h"
#endif
#include <limits>
extern mozilla::LogModule* GetMediaSourceLog();
#define MSE_DEBUG(arg, ...) \
DDMOZ_LOG(GetMediaSourceLog(), mozilla::LogLevel::Debug, "(%s)::%s: " arg, \
mType.OriginalString().Data(), __func__, ##__VA_ARGS__)
#define MSE_DEBUGV(arg, ...) \
DDMOZ_LOG(GetMediaSourceLog(), mozilla::LogLevel::Verbose, "(%s)::%s: " arg, \
mType.OriginalString().Data(), __func__, ##__VA_ARGS__)
mozilla::LogModule* GetMediaSourceSamplesLog() {
static mozilla::LazyLogModule sLogModule("MediaSourceSamples");
return sLogModule;
}
#define SAMPLE_DEBUG(arg, ...) \
DDMOZ_LOG(GetMediaSourceSamplesLog(), mozilla::LogLevel::Debug, \
"(%s)::%s: " arg, mType.OriginalString().Data(), __func__, \
##__VA_ARGS__)
namespace mozilla {
using dom::SourceBufferAppendMode;
using media::TimeInterval;
using media::TimeIntervals;
using media::TimeUnit;
typedef SourceBufferTask::AppendBufferResult AppendBufferResult;
typedef SourceBufferAttributes::AppendState AppendState;
static const char* AppendStateToStr(AppendState aState) {
switch (aState) {
case AppendState::WAITING_FOR_SEGMENT:
return "WAITING_FOR_SEGMENT";
case AppendState::PARSING_INIT_SEGMENT:
return "PARSING_INIT_SEGMENT";
case AppendState::PARSING_MEDIA_SEGMENT:
return "PARSING_MEDIA_SEGMENT";
default:
return "IMPOSSIBLE";
}
}
static Atomic<uint32_t> sStreamSourceID(0u);
class DispatchKeyNeededEvent : public Runnable {
public:
DispatchKeyNeededEvent(MediaSourceDecoder* aDecoder,
const nsTArray<uint8_t>& aInitData,
const nsString& aInitDataType)
: Runnable("DispatchKeyNeededEvent"),
mDecoder(aDecoder),
mInitData(aInitData.Clone()),
mInitDataType(aInitDataType) {}
NS_IMETHOD Run() override {
// Note: Null check the owner, as the decoder could have been shutdown
// since this event was dispatched.
MediaDecoderOwner* owner = mDecoder->GetOwner();
if (owner) {
owner->DispatchEncrypted(mInitData, mInitDataType);
}
mDecoder = nullptr;
return NS_OK;
}
private:
RefPtr<MediaSourceDecoder> mDecoder;
nsTArray<uint8_t> mInitData;
nsString mInitDataType;
};
TrackBuffersManager::TrackBuffersManager(MediaSourceDecoder* aParentDecoder,
const MediaContainerType& aType)
: mBufferFull(false),
mFirstInitializationSegmentReceived(false),
mChangeTypeReceived(false),
mNewMediaSegmentStarted(false),
mActiveTrack(false),
mType(aType),
mParser(ContainerParser::CreateForMIMEType(aType)),
mProcessedInput(0),
mParentDecoder(new nsMainThreadPtrHolder<MediaSourceDecoder>(
"TrackBuffersManager::mParentDecoder", aParentDecoder,
false /* strict */)),
mAbstractMainThread(aParentDecoder->AbstractMainThread()),
mEnded(false),
mVideoEvictionThreshold(Preferences::GetUint(
"media.mediasource.eviction_threshold.video", 100 * 1024 * 1024)),
mAudioEvictionThreshold(Preferences::GetUint(
"media.mediasource.eviction_threshold.audio", 20 * 1024 * 1024)),
mEvictionState(EvictionState::NO_EVICTION_NEEDED),
mMutex("TrackBuffersManager"),
mTaskQueue(aParentDecoder->GetDemuxer()->GetTaskQueue()) {
MOZ_ASSERT(NS_IsMainThread(), "Must be instanciated on the main thread");
DDLINKCHILD("parser", mParser.get());
}
TrackBuffersManager::~TrackBuffersManager() { ShutdownDemuxers(); }
RefPtr<TrackBuffersManager::AppendPromise> TrackBuffersManager::AppendData(
already_AddRefed<MediaByteBuffer> aData,
const SourceBufferAttributes& aAttributes) {
MOZ_ASSERT(NS_IsMainThread());
RefPtr<MediaByteBuffer> data(aData);
MSE_DEBUG("Appending %zu bytes", data->Length());
mEnded = false;
return InvokeAsync(static_cast<AbstractThread*>(GetTaskQueueSafe().get()),
this, __func__, &TrackBuffersManager::DoAppendData,
data.forget(), aAttributes);
}
RefPtr<TrackBuffersManager::AppendPromise> TrackBuffersManager::DoAppendData(
already_AddRefed<MediaByteBuffer> aData,
const SourceBufferAttributes& aAttributes) {
RefPtr<AppendBufferTask> task =
new AppendBufferTask(std::move(aData), aAttributes);
RefPtr<AppendPromise> p = task->mPromise.Ensure(__func__);
QueueTask(task);
return p;
}
void TrackBuffersManager::QueueTask(SourceBufferTask* aTask) {
// The source buffer is a wrapped native, it would be unlinked twice and so
// the TrackBuffersManager::Detach() would also be called twice. Since the
// detach task has been done before, we could ignore this task.
RefPtr<TaskQueue> taskQueue = GetTaskQueueSafe();
if (!taskQueue) {
MOZ_ASSERT(aTask->GetType() == SourceBufferTask::Type::Detach,
"only detach task could happen here!");
MSE_DEBUG("Could not queue the task '%s' without task queue",
aTask->GetTypeName());
return;
}
if (!taskQueue->IsCurrentThreadIn()) {
nsresult rv =
taskQueue->Dispatch(NewRunnableMethod<RefPtr<SourceBufferTask>>(
"TrackBuffersManager::QueueTask", this,
&TrackBuffersManager::QueueTask, aTask));
MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
Unused << rv;
return;
}
mQueue.Push(aTask);
ProcessTasks();
}
void TrackBuffersManager::ProcessTasks() {
// ProcessTask is always called OnTaskQueue, however it is possible that it is
// called once again after a first Detach task has run, in which case
// mTaskQueue would be null.
// This can happen under two conditions:
// 1- Two Detach tasks were queued in a row due to a double cycle collection.
// 2- An call to ProcessTasks() had queued another run of ProcessTasks while
// a Detach task is pending.
// We handle these two cases by aborting early.
// A second Detach task was queued, prior the first one running, ignore it.
if (!mTaskQueue) {
RefPtr<SourceBufferTask> task = mQueue.Pop();
if (!task) {
return;
}
MOZ_RELEASE_ASSERT(task->GetType() == SourceBufferTask::Type::Detach,
"only detach task could happen here!");
MSE_DEBUG("Could not process the task '%s' after detached",
task->GetTypeName());
return;
}
MOZ_ASSERT(OnTaskQueue());
typedef SourceBufferTask::Type Type;
if (mCurrentTask) {
// Already have a task pending. ProcessTask will be scheduled once the
// current task complete.
return;
}
RefPtr<SourceBufferTask> task = mQueue.Pop();
if (!task) {
// nothing to do.
return;
}
MSE_DEBUG("Process task '%s'", task->GetTypeName());
switch (task->GetType()) {
case Type::AppendBuffer:
mCurrentTask = task;
if (!mInputBuffer || mInputBuffer->IsEmpty()) {
// Note: we reset mInputBuffer here to ensure it doesn't grow unbounded.
mInputBuffer.reset();
mInputBuffer = Some(MediaSpan(task->As<AppendBufferTask>()->mBuffer));
} else if (!mInputBuffer->Append(task->As<AppendBufferTask>()->mBuffer)) {
RejectAppend(NS_ERROR_OUT_OF_MEMORY, __func__);
return;
}
mSourceBufferAttributes = MakeUnique<SourceBufferAttributes>(
task->As<AppendBufferTask>()->mAttributes);
mAppendWindow = TimeInterval(
TimeUnit::FromSeconds(
mSourceBufferAttributes->GetAppendWindowStart()),
TimeUnit::FromSeconds(mSourceBufferAttributes->GetAppendWindowEnd()));
ScheduleSegmentParserLoop();
break;
case Type::RangeRemoval: {
bool rv = CodedFrameRemoval(task->As<RangeRemovalTask>()->mRange);
task->As<RangeRemovalTask>()->mPromise.Resolve(rv, __func__);
break;
}
case Type::EvictData:
DoEvictData(task->As<EvictDataTask>()->mPlaybackTime,
task->As<EvictDataTask>()->mSizeToEvict);
break;
case Type::Abort:
// not handled yet, and probably never.
break;
case Type::Reset:
CompleteResetParserState();
break;
case Type::Detach:
mCurrentInputBuffer = nullptr;
MOZ_DIAGNOSTIC_ASSERT(mQueue.Length() == 0,
"Detach task must be the last");
mVideoTracks.Reset();
mAudioTracks.Reset();
ShutdownDemuxers();
ResetTaskQueue();
return;
case Type::ChangeType:
MOZ_RELEASE_ASSERT(!mCurrentTask);
mType = task->As<ChangeTypeTask>()->mType;
mChangeTypeReceived = true;
mInitData = nullptr;
// A new input buffer will be created once we receive a new init segment.
// The first segment received after a changeType call must be an init
// segment.
mCurrentInputBuffer = nullptr;
CompleteResetParserState();
break;
default:
NS_WARNING("Invalid Task");
}
TaskQueueFromTaskQueue()->Dispatch(
NewRunnableMethod("TrackBuffersManager::ProcessTasks", this,
&TrackBuffersManager::ProcessTasks));
}
// The MSE spec requires that we abort the current SegmentParserLoop
// which is then followed by a call to ResetParserState.
// However due to our asynchronous design this causes inherent difficulties.
// As the spec behaviour is non deterministic anyway, we instead process all
// pending frames found in the input buffer.
void TrackBuffersManager::AbortAppendData() {
MOZ_ASSERT(NS_IsMainThread());
MSE_DEBUG("");
QueueTask(new AbortTask());
}
void TrackBuffersManager::ResetParserState(
SourceBufferAttributes& aAttributes) {
MOZ_ASSERT(NS_IsMainThread());
MSE_DEBUG("");
// Spec states:
// 1. If the append state equals PARSING_MEDIA_SEGMENT and the input buffer
// contains some complete coded frames, then run the coded frame processing
// algorithm until all of these complete coded frames have been processed.
// However, we will wait until all coded frames have been processed regardless
// of the value of append state.
QueueTask(new ResetTask());
// ResetParserState has some synchronous steps that much be performed now.
// The remaining steps will be performed once the ResetTask gets executed.
// 6. If the mode attribute equals "sequence", then set the group start
// timestamp to the group end timestamp
if (aAttributes.GetAppendMode() == SourceBufferAppendMode::Sequence) {
aAttributes.SetGroupStartTimestamp(aAttributes.GetGroupEndTimestamp());
}
// 8. Set append state to WAITING_FOR_SEGMENT.
aAttributes.SetAppendState(AppendState::WAITING_FOR_SEGMENT);
}
RefPtr<TrackBuffersManager::RangeRemovalPromise>
TrackBuffersManager::RangeRemoval(TimeUnit aStart, TimeUnit aEnd) {
MOZ_ASSERT(NS_IsMainThread());
MSE_DEBUG("From %.2f to %.2f", aStart.ToSeconds(), aEnd.ToSeconds());
mEnded = false;
return InvokeAsync(static_cast<AbstractThread*>(GetTaskQueueSafe().get()),
this, __func__,
&TrackBuffersManager::CodedFrameRemovalWithPromise,
TimeInterval(aStart, aEnd));
}
TrackBuffersManager::EvictDataResult TrackBuffersManager::EvictData(
const TimeUnit& aPlaybackTime, int64_t aSize) {
MOZ_ASSERT(NS_IsMainThread());
if (aSize > EvictionThreshold()) {
// We're adding more data than we can hold.
return EvictDataResult::BUFFER_FULL;
}
const int64_t toEvict = GetSize() + aSize - EvictionThreshold();
const uint32_t canEvict =
Evictable(HasVideo() ? TrackInfo::kVideoTrack : TrackInfo::kAudioTrack);
MSE_DEBUG("currentTime=%" PRId64 " buffered=%" PRId64
"kB, eviction threshold=%" PRId64
"kB, "
"evict=%" PRId64 "kB canevict=%" PRIu32 "kB",
aPlaybackTime.ToMicroseconds(), GetSize() / 1024,
EvictionThreshold() / 1024, toEvict / 1024, canEvict / 1024);
if (toEvict <= 0) {
mEvictionState = EvictionState::NO_EVICTION_NEEDED;
return EvictDataResult::NO_DATA_EVICTED;
}
EvictDataResult result;
if (mBufferFull && mEvictionState == EvictionState::EVICTION_COMPLETED &&
canEvict < uint32_t(toEvict)) {
// Our buffer is currently full. We will make another eviction attempt.
// However, the current appendBuffer will fail as we can't know ahead of
// time if the eviction will later succeed.
result = EvictDataResult::BUFFER_FULL;
} else {
mEvictionState = EvictionState::EVICTION_NEEDED;
result = EvictDataResult::NO_DATA_EVICTED;
}
MSE_DEBUG("Reached our size limit, schedule eviction of %" PRId64
" bytes (%s)",
toEvict,
result == EvictDataResult::BUFFER_FULL ? "buffer full"
: "no data evicted");
QueueTask(new EvictDataTask(aPlaybackTime, toEvict));
return result;
}
void TrackBuffersManager::ChangeType(const MediaContainerType& aType) {
MOZ_ASSERT(NS_IsMainThread());
QueueTask(new ChangeTypeTask(aType));
}
TimeIntervals TrackBuffersManager::Buffered() const {
MSE_DEBUG("");
MutexAutoLock mut(mMutex);
nsTArray<const TimeIntervals*> tracks;
if (HasVideo()) {
tracks.AppendElement(&mVideoBufferedRanges);
}
if (HasAudio()) {
tracks.AppendElement(&mAudioBufferedRanges);
}
// 2. Let highest end time be the largest track buffer ranges end time across
// all the track buffers managed by this SourceBuffer object.
TimeUnit highestEndTime = HighestEndTime(tracks);
// 3. Let intersection ranges equal a TimeRange object containing a single
// range from 0 to highest end time.
TimeIntervals intersection{
TimeInterval(TimeUnit::FromSeconds(0), highestEndTime)};
// 4. For each track buffer managed by this SourceBuffer, run the following
// steps:
// 1. Let track ranges equal the track buffer ranges for the current track
// buffer.
for (const TimeIntervals* trackRanges : tracks) {
// 2. If readyState is "ended", then set the end time on the last range in
// track ranges to highest end time.
// 3. Let new intersection ranges equal the intersection between the
// intersection ranges and the track ranges.
if (mEnded) {
TimeIntervals tR = *trackRanges;
tR.Add(TimeInterval(tR.GetEnd(), highestEndTime));
intersection.Intersection(tR);
} else {
intersection.Intersection(*trackRanges);
}
}
return intersection;
}
int64_t TrackBuffersManager::GetSize() const { return mSizeSourceBuffer; }
void TrackBuffersManager::Ended() { mEnded = true; }
void TrackBuffersManager::Detach() {
MOZ_ASSERT(NS_IsMainThread());
MSE_DEBUG("");
QueueTask(new DetachTask());
}
void TrackBuffersManager::CompleteResetParserState() {
MOZ_ASSERT(OnTaskQueue());
MSE_DEBUG("");
// We shouldn't change mInputDemuxer while a demuxer init/reset request is
// being processed. See bug 1239983.
MOZ_DIAGNOSTIC_ASSERT(!mDemuxerInitRequest.Exists(),
"Previous AppendBuffer didn't complete");
for (auto& track : GetTracksList()) {
// 2. Unset the last decode timestamp on all track buffers.
// 3. Unset the last frame duration on all track buffers.
// 4. Unset the highest end timestamp on all track buffers.
// 5. Set the need random access point flag on all track buffers to true.
track->ResetAppendState();
// if we have been aborted, we may have pending frames that we are going
// to discard now.
track->mQueuedSamples.Clear();
}
// 7. Remove all bytes from the input buffer.
mPendingInputBuffer.reset();
mInputBuffer.reset();
if (mCurrentInputBuffer) {
mCurrentInputBuffer->EvictAll();
// The demuxer will be recreated during the next run of SegmentParserLoop.
// As such we don't need to notify it that data has been removed.
mCurrentInputBuffer = new SourceBufferResource();
}
// We could be left with a demuxer in an unusable state. It needs to be
// recreated. Unless we have a pending changeType operation, we store in the
// InputBuffer an init segment which will be parsed during the next Segment
// Parser Loop and a new demuxer will be created and initialized.
// If we are in the middle of a changeType operation, then we do not have an
// init segment yet. The next appendBuffer operation will need to provide such
// init segment.
if (mFirstInitializationSegmentReceived && !mChangeTypeReceived) {
MOZ_ASSERT(mInitData && mInitData->Length(),
"we must have an init segment");
// The aim here is really to destroy our current demuxer.
CreateDemuxerforMIMEType();
// Recreate our input buffer. We can't directly assign the initData buffer
// to mInputBuffer as it will get modified in the Segment Parser Loop.
mInputBuffer = Some(MediaSpan::WithCopyOf(mInitData));
RecreateParser(true);
} else {
RecreateParser(false);
}
}
int64_t TrackBuffersManager::EvictionThreshold() const {
if (HasVideo()) {
return mVideoEvictionThreshold;
}
return mAudioEvictionThreshold;
}
void TrackBuffersManager::DoEvictData(const TimeUnit& aPlaybackTime,
int64_t aSizeToEvict) {
MOZ_ASSERT(OnTaskQueue());
mEvictionState = EvictionState::EVICTION_COMPLETED;
// Video is what takes the most space, only evict there if we have video.
auto& track = HasVideo() ? mVideoTracks : mAudioTracks;
const auto& buffer = track.GetTrackBuffer();
if (buffer.IsEmpty()) {
// Buffer has been emptied while the eviction was queued, nothing to do.
return;
}
if (track.mBufferedRanges.IsEmpty()) {
MSE_DEBUG(
"DoEvictData running with no buffered ranges. 0 duration data likely "
"present in our buffer(s). Evicting all data!");
// We have no buffered ranges, but may still have data. This happens if the
// buffer is full of 0 duration data. Normal removal procedures don't clear
// 0 duration data, so blow away all our data.
RemoveAllCodedFrames();
return;
}
// Remove any data we've already played, or before the next sample to be
// demuxed whichever is lowest.
TimeUnit lowerLimit = std::min(track.mNextSampleTime, aPlaybackTime);
uint32_t lastKeyFrameIndex = 0;
int64_t toEvict = aSizeToEvict;
int64_t partialEvict = 0;
for (uint32_t i = 0; i < buffer.Length(); i++) {
const auto& frame = buffer[i];
if (frame->mKeyframe) {
lastKeyFrameIndex = i;
toEvict -= partialEvict;
if (toEvict < 0) {
break;
}
partialEvict = 0;
}
if (frame->GetEndTime() >= lowerLimit) {
break;
}
partialEvict += frame->ComputedSizeOfIncludingThis();
}
const int64_t finalSize = mSizeSourceBuffer - aSizeToEvict;
if (lastKeyFrameIndex > 0) {
MSE_DEBUG("Step1. Evicting %" PRId64 " bytes prior currentTime",
aSizeToEvict - toEvict);
TimeUnit start = track.mBufferedRanges[0].mStart;
TimeUnit end =
buffer[lastKeyFrameIndex]->mTime - TimeUnit::FromMicroseconds(1);
if (end > start) {
CodedFrameRemoval(TimeInterval(start, end));
}
}
if (mSizeSourceBuffer <= finalSize) {
return;
}
toEvict = mSizeSourceBuffer - finalSize;
// See if we can evict data into the future.
// We do not evict data from the currently used buffered interval.
TimeUnit currentPosition = std::max(aPlaybackTime, track.mNextSampleTime);
TimeIntervals futureBuffered(
TimeInterval(currentPosition, TimeUnit::FromInfinity()));
futureBuffered.Intersection(track.mBufferedRanges);
futureBuffered.SetFuzz(MediaSourceDemuxer::EOS_FUZZ / 2);
if (futureBuffered.Length() <= 1) {
// We have one continuous segment ahead of us:
// nothing further can be evicted.
return;
}
// Don't evict before the end of the current segment
TimeUnit upperLimit = futureBuffered[0].mEnd;
uint32_t evictedFramesStartIndex = buffer.Length();
for (int32_t i = buffer.Length() - 1; i >= 0; i--) {
const auto& frame = buffer[i];
if (frame->mTime <= upperLimit || toEvict < 0) {
// We've reached a frame that shouldn't be evicted -> Evict after it ->
// i+1. Or the previous loop reached the eviction threshold -> Evict from
// it -> i+1.
evictedFramesStartIndex = i + 1;
break;
}
toEvict -= frame->ComputedSizeOfIncludingThis();
}
if (evictedFramesStartIndex < buffer.Length()) {
MSE_DEBUG("Step2. Evicting %" PRId64 " bytes from trailing data",
mSizeSourceBuffer - finalSize - toEvict);
CodedFrameRemoval(TimeInterval(buffer[evictedFramesStartIndex]->mTime,
TimeUnit::FromInfinity()));
}
}
RefPtr<TrackBuffersManager::RangeRemovalPromise>
TrackBuffersManager::CodedFrameRemovalWithPromise(TimeInterval aInterval) {
MOZ_ASSERT(OnTaskQueue());
RefPtr<RangeRemovalTask> task = new RangeRemovalTask(aInterval);
RefPtr<RangeRemovalPromise> p = task->mPromise.Ensure(__func__);
QueueTask(task);
return p;
}
bool TrackBuffersManager::CodedFrameRemoval(TimeInterval aInterval) {
MOZ_ASSERT(OnTaskQueue());
MSE_DEBUG("From %.2fs to %.2f", aInterval.mStart.ToSeconds(),
aInterval.mEnd.ToSeconds());
#if DEBUG
if (HasVideo()) {
MSE_DEBUG("before video ranges=%s",
DumpTimeRanges(mVideoTracks.mBufferedRanges).get());
}
if (HasAudio()) {
MSE_DEBUG("before audio ranges=%s",
DumpTimeRanges(mAudioTracks.mBufferedRanges).get());
}
#endif
// 1. Let start be the starting presentation timestamp for the removal range.
TimeUnit start = aInterval.mStart;
// 2. Let end be the end presentation timestamp for the removal range.
TimeUnit end = aInterval.mEnd;
bool dataRemoved = false;
// 3. For each track buffer in this source buffer, run the following steps:
for (auto track : GetTracksList()) {
MSE_DEBUGV("Processing %s track", track->mInfo->mMimeType.get());
// 1. Let remove end timestamp be the current value of duration
// At worse we will remove all frames until the end, unless a key frame is
// found between the current interval's end and the trackbuffer's end.
TimeUnit removeEndTimestamp = track->mBufferedRanges.GetEnd();
if (start > removeEndTimestamp) {
// Nothing to remove.
continue;
}
// 2. If this track buffer has a random access point timestamp that is
// greater than or equal to end, then update remove end timestamp to that
// random access point timestamp.
if (end < track->mBufferedRanges.GetEnd()) {
for (auto& frame : track->GetTrackBuffer()) {
if (frame->mKeyframe && frame->mTime >= end) {
removeEndTimestamp = frame->mTime;
break;
}
}
}
// 3. Remove all media data, from this track buffer, that contain starting
// timestamps greater than or equal to start and less than the remove end
// timestamp.
// 4. Remove decoding dependencies of the coded frames removed in the
// previous step: Remove all coded frames between the coded frames removed
// in the previous step and the next random access point after those removed
// frames.
TimeIntervals removedInterval{TimeInterval(start, removeEndTimestamp)};
RemoveFrames(removedInterval, *track, 0, RemovalMode::kRemoveFrame);
// 5. If this object is in activeSourceBuffers, the current playback
// position is greater than or equal to start and less than the remove end
// timestamp, and HTMLMediaElement.readyState is greater than HAVE_METADATA,
// then set the HTMLMediaElement.readyState attribute to HAVE_METADATA and
// stall playback. This will be done by the MDSM during playback.
// TODO properly, so it works even if paused.
}
UpdateBufferedRanges();
// Update our reported total size.
mSizeSourceBuffer = mVideoTracks.mSizeBuffer + mAudioTracks.mSizeBuffer;
// 4. If buffer full flag equals true and this object is ready to accept more
// bytes, then set the buffer full flag to false.
if (mBufferFull && mSizeSourceBuffer < EvictionThreshold()) {
mBufferFull = false;
}
return dataRemoved;
}
void TrackBuffersManager::RemoveAllCodedFrames() {
// This is similar to RemoveCodedFrames, but will attempt to remove ALL
// the frames. This is not to spec, as explained below at step 3.1. Steps
// below coincide with Remove Coded Frames algorithm from the spec.
MSE_DEBUG("RemoveAllCodedFrames called.");
MOZ_ASSERT(OnTaskQueue());
// 1. Let start be the starting presentation timestamp for the removal range.
TimeUnit start{};
// 2. Let end be the end presentation timestamp for the removal range.
TimeUnit end = TimeUnit::FromMicroseconds(1);
// Find an end time such that our range will include every frame in every
// track. We do this by setting the end of our interval to the largest end
// time seen + 1 microsecond.
for (TrackData* track : GetTracksList()) {
for (auto& frame : track->GetTrackBuffer()) {
MOZ_ASSERT(frame->mTime >= start,
"Shouldn't have frame at negative time!");
TimeUnit frameEnd = frame->mTime + frame->mDuration;
if (frameEnd > end) {
end = frameEnd + TimeUnit::FromMicroseconds(1);
}
}
}
// 3. For each track buffer in this source buffer, run the following steps:
TimeIntervals removedInterval{TimeInterval(start, end)};
for (TrackData* track : GetTracksList()) {
// 1. Let remove end timestamp be the current value of duration
// ^ It's off spec, but we ignore this in order to clear 0 duration frames.
// If we don't ignore this rule and our buffer is full of 0 duration frames
// at timestamp n, we get an eviction range of [0, n). When we get to step
// 3.3 below, the 0 duration frames will not be evicted because their
// timestamp is not less than remove end timestamp -- it will in fact be
// equal to remove end timestamp.
//
// 2. If this track buffer has a random access point timestamp that is
// greater than or equal to end, then update remove end timestamp to that
// random access point timestamp.
// ^ We've made sure end > any sample's timestamp, so can skip this.
//
// 3. Remove all media data, from this track buffer, that contain starting
// timestamps greater than or equal to start and less than the remove end
// timestamp.
// 4. Remove decoding dependencies of the coded frames removed in the
// previous step: Remove all coded frames between the coded frames removed
// in the previous step and the next random access point after those removed
// frames.
// This should remove every frame in the track because removedInterval was
// constructed such that every frame in any track falls into that interval.
RemoveFrames(removedInterval, *track, 0, RemovalMode::kRemoveFrame);
// 5. If this object is in activeSourceBuffers, the current playback
// position is greater than or equal to start and less than the remove end
// timestamp, and HTMLMediaElement.readyState is greater than HAVE_METADATA,
// then set the HTMLMediaElement.readyState attribute to HAVE_METADATA and
// stall playback. This will be done by the MDSM during playback.
// TODO properly, so it works even if paused.
}
UpdateBufferedRanges();
MOZ_ASSERT(mAudioBufferedRanges.IsEmpty(),
"Should have no buffered video ranges after evicting everything.");
MOZ_ASSERT(mVideoBufferedRanges.IsEmpty(),
"Should have no buffered video ranges after evicting everything.");
mSizeSourceBuffer = mVideoTracks.mSizeBuffer + mAudioTracks.mSizeBuffer;
MOZ_ASSERT(mSizeSourceBuffer == 0,
"Buffer should be empty after evicting everything!");
if (mBufferFull && mSizeSourceBuffer < EvictionThreshold()) {
mBufferFull = false;
}
}
void TrackBuffersManager::UpdateBufferedRanges() {
MutexAutoLock mut(mMutex);
mVideoBufferedRanges = mVideoTracks.mSanitizedBufferedRanges;
mAudioBufferedRanges = mAudioTracks.mSanitizedBufferedRanges;
#if DEBUG
if (HasVideo()) {
MSE_DEBUG("after video ranges=%s",
DumpTimeRanges(mVideoTracks.mBufferedRanges).get());
}
if (HasAudio()) {
MSE_DEBUG("after audio ranges=%s",
DumpTimeRanges(mAudioTracks.mBufferedRanges).get());
}
#endif
}
void TrackBuffersManager::SegmentParserLoop() {
MOZ_ASSERT(OnTaskQueue());
while (true) {
// 1. If the input buffer is empty, then jump to the need more data step
// below.
if (!mInputBuffer || mInputBuffer->IsEmpty()) {
NeedMoreData();
return;
}
// 2. If the input buffer contains bytes that violate the SourceBuffer
// byte stream format specification, then run the append error algorithm
// with the decode error parameter set to true and abort this algorithm.
// TODO
// 3. Remove any bytes that the byte stream format specifications say must
// be ignored from the start of the input buffer. We do not remove bytes
// from our input buffer. Instead we enforce that our ContainerParser is
// able to skip over all data that is supposed to be ignored.
// 4. If the append state equals WAITING_FOR_SEGMENT, then run the following
// steps:
if (mSourceBufferAttributes->GetAppendState() ==
AppendState::WAITING_FOR_SEGMENT) {
MediaResult haveInitSegment =
mParser->IsInitSegmentPresent(*mInputBuffer);
if (NS_SUCCEEDED(haveInitSegment)) {
SetAppendState(AppendState::PARSING_INIT_SEGMENT);
if (mFirstInitializationSegmentReceived && !mChangeTypeReceived) {
// This is a new initialization segment. Obsolete the old one.
RecreateParser(false);
}
continue;
}
MediaResult haveMediaSegment =
mParser->IsMediaSegmentPresent(*mInputBuffer);
if (NS_SUCCEEDED(haveMediaSegment)) {
SetAppendState(AppendState::PARSING_MEDIA_SEGMENT);
mNewMediaSegmentStarted = true;
continue;
}
// We have neither an init segment nor a media segment.
// Check if it was invalid data.
if (haveInitSegment != NS_ERROR_NOT_AVAILABLE) {
MSE_DEBUG("Found invalid data.");
RejectAppend(haveInitSegment, __func__);
return;
}
if (haveMediaSegment != NS_ERROR_NOT_AVAILABLE) {
MSE_DEBUG("Found invalid data.");
RejectAppend(haveMediaSegment, __func__);
return;
}
MSE_DEBUG("Found incomplete data.");
NeedMoreData();
return;
}
MOZ_ASSERT(mSourceBufferAttributes->GetAppendState() ==
AppendState::PARSING_INIT_SEGMENT ||
mSourceBufferAttributes->GetAppendState() ==
AppendState::PARSING_MEDIA_SEGMENT);
int64_t start = 0;
int64_t end = 0;
MediaResult newData = NS_ERROR_NOT_AVAILABLE;
if (mSourceBufferAttributes->GetAppendState() ==
AppendState::PARSING_INIT_SEGMENT ||
(mSourceBufferAttributes->GetAppendState() ==
AppendState::PARSING_MEDIA_SEGMENT &&
mFirstInitializationSegmentReceived && !mChangeTypeReceived)) {
newData = mParser->ParseStartAndEndTimestamps(*mInputBuffer, start, end);
if (NS_FAILED(newData) && newData.Code() != NS_ERROR_NOT_AVAILABLE) {
RejectAppend(newData, __func__);
return;
}
mProcessedInput += mInputBuffer->Length();
}
// 5. If the append state equals PARSING_INIT_SEGMENT, then run the
// following steps:
if (mSourceBufferAttributes->GetAppendState() ==
AppendState::PARSING_INIT_SEGMENT) {
if (mParser->InitSegmentRange().IsEmpty()) {
mInputBuffer.reset();
NeedMoreData();
return;
}
InitializationSegmentReceived();
return;
}
if (mSourceBufferAttributes->GetAppendState() ==
AppendState::PARSING_MEDIA_SEGMENT) {
// 1. If the first initialization segment received flag is false, then run
// the append error algorithm with the decode error parameter set to
// true and abort this algorithm.
// Or we are in the process of changeType, in which case we must first
// get an init segment before getting a media segment.
if (!mFirstInitializationSegmentReceived || mChangeTypeReceived) {
RejectAppend(NS_ERROR_FAILURE, __func__);
return;
}
// We can't feed some demuxers (WebMDemuxer) with data that do not have
// monotonizally increasing timestamps. So we check if we have a
// discontinuity from the previous segment parsed.
// If so, recreate a new demuxer to ensure that the demuxer is only fed
// monotonically increasing data.
if (mNewMediaSegmentStarted) {
if (NS_SUCCEEDED(newData) && mLastParsedEndTime.isSome() &&
start < mLastParsedEndTime.ref().ToMicroseconds()) {
MSE_DEBUG("Re-creating demuxer");
ResetDemuxingState();
return;
}
if (NS_SUCCEEDED(newData) || !mParser->MediaSegmentRange().IsEmpty()) {
if (mPendingInputBuffer) {
// We now have a complete media segment header. We can resume
// parsing the data.
AppendDataToCurrentInputBuffer(*mPendingInputBuffer);
mPendingInputBuffer.reset();
}
mNewMediaSegmentStarted = false;
} else {
// We don't have any data to demux yet, stash aside the data.
// This also handles the case:
// 2. If the input buffer does not contain a complete media segment
// header yet, then jump to the need more data step below.
if (!mPendingInputBuffer) {
mPendingInputBuffer = Some(MediaSpan(*mInputBuffer));
} else {
// Note we reset mInputBuffer below, so this won't end up appending
// the contents of mInputBuffer to itself.
mPendingInputBuffer->Append(*mInputBuffer);
}
mInputBuffer.reset();
NeedMoreData();
return;
}
}
// 3. If the input buffer contains one or more complete coded frames, then
// run the coded frame processing algorithm.
RefPtr<TrackBuffersManager> self = this;
CodedFrameProcessing()
->Then(
TaskQueueFromTaskQueue(), __func__,
[self](bool aNeedMoreData) {
self->mProcessingRequest.Complete();
if (aNeedMoreData) {
self->NeedMoreData();
} else {
self->ScheduleSegmentParserLoop();
}
},
[self](const MediaResult& aRejectValue) {
self->mProcessingRequest.Complete();
self->RejectAppend(aRejectValue, __func__);
})
->Track(mProcessingRequest);
return;
}
}
}
void TrackBuffersManager::NeedMoreData() {
MSE_DEBUG("");
MOZ_DIAGNOSTIC_ASSERT(mCurrentTask &&
mCurrentTask->GetType() ==
SourceBufferTask::Type::AppendBuffer);
MOZ_DIAGNOSTIC_ASSERT(mSourceBufferAttributes);
mCurrentTask->As<AppendBufferTask>()->mPromise.Resolve(
SourceBufferTask::AppendBufferResult(mActiveTrack,
*mSourceBufferAttributes),
__func__);
mSourceBufferAttributes = nullptr;
mCurrentTask = nullptr;
ProcessTasks();
}
void TrackBuffersManager::RejectAppend(const MediaResult& aRejectValue,
const char* aName) {
MSE_DEBUG("rv=%" PRIu32, static_cast<uint32_t>(aRejectValue.Code()));
MOZ_DIAGNOSTIC_ASSERT(mCurrentTask &&
mCurrentTask->GetType() ==
SourceBufferTask::Type::AppendBuffer);
mCurrentTask->As<AppendBufferTask>()->mPromise.Reject(aRejectValue, __func__);
mSourceBufferAttributes = nullptr;
mCurrentTask = nullptr;
ProcessTasks();
}
void TrackBuffersManager::ScheduleSegmentParserLoop() {
MOZ_ASSERT(OnTaskQueue());
TaskQueueFromTaskQueue()->Dispatch(
NewRunnableMethod("TrackBuffersManager::SegmentParserLoop", this,
&TrackBuffersManager::SegmentParserLoop));
}
void TrackBuffersManager::ShutdownDemuxers() {
if (mVideoTracks.mDemuxer) {
mVideoTracks.mDemuxer->BreakCycles();
mVideoTracks.mDemuxer = nullptr;
}
if (mAudioTracks.mDemuxer) {
mAudioTracks.mDemuxer->BreakCycles();
mAudioTracks.mDemuxer = nullptr;
}
// We shouldn't change mInputDemuxer while a demuxer init/reset request is
// being processed. See bug 1239983.
MOZ_DIAGNOSTIC_ASSERT(!mDemuxerInitRequest.Exists());
mInputDemuxer = nullptr;
mLastParsedEndTime.reset();
}
void TrackBuffersManager::CreateDemuxerforMIMEType() {
ShutdownDemuxers();
if (mType.Type() == MEDIAMIMETYPE(VIDEO_WEBM) ||
mType.Type() == MEDIAMIMETYPE(AUDIO_WEBM)) {
mInputDemuxer =
new WebMDemuxer(mCurrentInputBuffer, true /* IsMediaSource*/);
DDLINKCHILD("demuxer", mInputDemuxer.get());
return;
}
#ifdef MOZ_FMP4
if (mType.Type() == MEDIAMIMETYPE(VIDEO_MP4) ||
mType.Type() == MEDIAMIMETYPE(AUDIO_MP4)) {
mInputDemuxer = new MP4Demuxer(mCurrentInputBuffer);
DDLINKCHILD("demuxer", mInputDemuxer.get());
return;
}
#endif
NS_WARNING("Not supported (yet)");
}
// We reset the demuxer by creating a new one and initializing it.
void TrackBuffersManager::ResetDemuxingState() {
MOZ_ASSERT(OnTaskQueue());
MOZ_ASSERT(mParser && mParser->HasInitData());
RecreateParser(true);
mCurrentInputBuffer = new SourceBufferResource();
// The demuxer isn't initialized yet ; we don't want to notify it
// that data has been appended yet ; so we simply append the init segment
// to the resource.
mCurrentInputBuffer->AppendData(mParser->InitData());
CreateDemuxerforMIMEType();
if (!mInputDemuxer) {
RejectAppend(NS_ERROR_FAILURE, __func__);
return;
}
mInputDemuxer->Init()
->Then(TaskQueueFromTaskQueue(), __func__, this,
&TrackBuffersManager::OnDemuxerResetDone,
&TrackBuffersManager::OnDemuxerInitFailed)
->Track(mDemuxerInitRequest);
}
void TrackBuffersManager::OnDemuxerResetDone(const MediaResult& aResult) {
MOZ_ASSERT(OnTaskQueue());
mDemuxerInitRequest.Complete();
if (NS_FAILED(aResult) && StaticPrefs::media_playback_warnings_as_errors()) {
RejectAppend(aResult, __func__);
return;
}
// mInputDemuxer shouldn't have been destroyed while a demuxer init/reset
// request was being processed. See bug 1239983.
MOZ_DIAGNOSTIC_ASSERT(mInputDemuxer);
if (aResult != NS_OK && mParentDecoder) {
RefPtr<TrackBuffersManager> self = this;
mAbstractMainThread->Dispatch(NS_NewRunnableFunction(
"TrackBuffersManager::OnDemuxerResetDone", [self, aResult]() {
if (self->mParentDecoder && self->mParentDecoder->GetOwner()) {
self->mParentDecoder->GetOwner()->DecodeWarning(aResult);
}
}));
}
// Recreate track demuxers.
uint32_t numVideos = mInputDemuxer->GetNumberTracks(TrackInfo::kVideoTrack);
if (numVideos) {
// We currently only handle the first video track.
mVideoTracks.mDemuxer =
mInputDemuxer->GetTrackDemuxer(TrackInfo::kVideoTrack, 0);
MOZ_ASSERT(mVideoTracks.mDemuxer);
DDLINKCHILD("video demuxer", mVideoTracks.mDemuxer.get());
}
uint32_t numAudios = mInputDemuxer->GetNumberTracks(TrackInfo::kAudioTrack);
if (numAudios) {
// We currently only handle the first audio track.
mAudioTracks.mDemuxer =
mInputDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0);
MOZ_ASSERT(mAudioTracks.mDemuxer);
DDLINKCHILD("audio demuxer", mAudioTracks.mDemuxer.get());
}
if (mPendingInputBuffer) {
// We had a partial media segment header stashed aside.
// Reparse its content so we can continue parsing the current input buffer.
int64_t start, end;
mParser->ParseStartAndEndTimestamps(*mPendingInputBuffer, start, end);
mProcessedInput += mPendingInputBuffer->Length();
}
SegmentParserLoop();
}
void TrackBuffersManager::AppendDataToCurrentInputBuffer(
const MediaSpan& aData) {
MOZ_ASSERT(mCurrentInputBuffer);
mCurrentInputBuffer->AppendData(aData);
mInputDemuxer->NotifyDataArrived();
}
void TrackBuffersManager::InitializationSegmentReceived() {
MOZ_ASSERT(OnTaskQueue());
MOZ_ASSERT(mParser->HasCompleteInitData());
int64_t endInit = mParser->InitSegmentRange().mEnd;
if (mInputBuffer->Length() > mProcessedInput ||
int64_t(mProcessedInput - mInputBuffer->Length()) > endInit) {
// Something is not quite right with the data appended. Refuse it.
RejectAppend(MediaResult(NS_ERROR_FAILURE,
"Invalid state following initialization segment"),
__func__);
return;
}
mCurrentInputBuffer = new SourceBufferResource();
// The demuxer isn't initialized yet ; we don't want to notify it
// that data has been appended yet ; so we simply append the init segment
// to the resource.
mCurrentInputBuffer->AppendData(mParser->InitData());
uint32_t length = endInit - (mProcessedInput - mInputBuffer->Length());
MOZ_RELEASE_ASSERT(length <= mInputBuffer->Length());
mInputBuffer->RemoveFront(length);
CreateDemuxerforMIMEType();
if (!mInputDemuxer) {
NS_WARNING("TODO type not supported");
RejectAppend(NS_ERROR_DOM_NOT_SUPPORTED_ERR, __func__);
return;
}
mInputDemuxer->Init()
->Then(TaskQueueFromTaskQueue(), __func__, this,
&TrackBuffersManager::OnDemuxerInitDone,
&TrackBuffersManager::OnDemuxerInitFailed)
->Track(mDemuxerInitRequest);
}
void TrackBuffersManager::OnDemuxerInitDone(const MediaResult& aResult) {
MOZ_ASSERT(OnTaskQueue());
MOZ_DIAGNOSTIC_ASSERT(mInputDemuxer, "mInputDemuxer has been destroyed");
mDemuxerInitRequest.Complete();
if (NS_FAILED(aResult) && StaticPrefs::media_playback_warnings_as_errors()) {
RejectAppend(aResult, __func__);
return;
}
MediaInfo info;
uint32_t numVideos = mInputDemuxer->GetNumberTracks(TrackInfo::kVideoTrack);
if (numVideos) {
// We currently only handle the first video track.
mVideoTracks.mDemuxer =
mInputDemuxer->GetTrackDemuxer(TrackInfo::kVideoTrack, 0);
MOZ_ASSERT(mVideoTracks.mDemuxer);
DDLINKCHILD("video demuxer", mVideoTracks.mDemuxer.get());
info.mVideo = *mVideoTracks.mDemuxer->GetInfo()->GetAsVideoInfo();
info.mVideo.mTrackId = 2;
}
uint32_t numAudios = mInputDemuxer->GetNumberTracks(TrackInfo::kAudioTrack);
if (numAudios) {
// We currently only handle the first audio track.
mAudioTracks.mDemuxer =
mInputDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0);
MOZ_ASSERT(mAudioTracks.mDemuxer);
DDLINKCHILD("audio demuxer", mAudioTracks.mDemuxer.get());
info.mAudio = *mAudioTracks.mDemuxer->GetInfo()->GetAsAudioInfo();
info.mAudio.mTrackId = 1;
}
int64_t videoDuration =
numVideos ? info.mVideo.mDuration.ToMicroseconds() : 0;
int64_t audioDuration =
numAudios ? info.mAudio.mDuration.ToMicroseconds() : 0;
int64_t duration = std::max(videoDuration, audioDuration);
// 1. Update the duration attribute if it currently equals NaN.
// Those steps are performed by the MediaSourceDecoder::SetInitialDuration
mAbstractMainThread->Dispatch(NewRunnableMethod<int64_t>(
"MediaSourceDecoder::SetInitialDuration", mParentDecoder.get(),
&MediaSourceDecoder::SetInitialDuration, duration ? duration : -1));
// 2. If the initialization segment has no audio, video, or text tracks, then
// run the append error algorithm with the decode error parameter set to true
// and abort these steps.
if (!numVideos && !numAudios) {
RejectAppend(NS_ERROR_FAILURE, __func__);
return;
}
// 3. If the first initialization segment received flag is true, then run the
// following steps:
if (mFirstInitializationSegmentReceived) {
if (numVideos != mVideoTracks.mNumTracks ||
numAudios != mAudioTracks.mNumTracks) {
RejectAppend(NS_ERROR_FAILURE, __func__);
return;
}
// 1. If more than one track for a single type are present (ie 2 audio
// tracks), then the Track IDs match the ones in the first initialization
// segment.
// TODO
// 2. Add the appropriate track descriptions from this initialization
// segment to each of the track buffers.
// TODO
// 3. Set the need random access point flag on all track buffers to true.
mVideoTracks.mNeedRandomAccessPoint = true;
mAudioTracks.mNeedRandomAccessPoint = true;
}
// Check if we've received the same init data again. Some streams will
// resend the same data. In these cases we don't need to change the stream
// id as it's the same stream. Doing so would recreate decoders, possibly
// leading to gaps in audio and/or video (see bug 1450952).
//
// It's possible to have the different binary representations for the same
// logical init data. If this case is encountered in the wild then these
// checks could be revised to compare MediaInfo rather than init segment
// bytes.
// For audio only source buffer we can check that only the AudioInfo has
// changed.
bool isRepeatInitData =
!mChangeTypeReceived && mInitData &&
((*mInitData.get() == *mParser->InitData()) ||
(numVideos == 0 && numAudios > 0 && mAudioTracks.mLastInfo &&
*mAudioTracks.mLastInfo->GetAsAudioInfo() == info.mAudio));
MOZ_ASSERT(mFirstInitializationSegmentReceived || !isRepeatInitData,
"Should never detect repeat init data for first segment!");
// If we have new init data we configure and set track info as needed. If we
// have repeat init data we carry forward our existing track info.
if (!isRepeatInitData) {
// Increase our stream id.
uint32_t streamID = sStreamSourceID++;
// 4. Let active track flag equal false.
bool activeTrack = false;
// 5. If the first initialization segment received flag is false, then run
// the following steps:
if (!mFirstInitializationSegmentReceived) {
mAudioTracks.mNumTracks = numAudios;
// TODO:
// 1. If the initialization segment contains tracks with codecs the user
// agent does not support, then run the append error algorithm with the
// decode error parameter set to true and abort these steps.
// 2. For each audio track in the initialization segment, run following
// steps: for (uint32_t i = 0; i < numAudios; i++) {
if (numAudios) {
// 1. Let audio byte stream track ID be the Track ID for the current
// track being processed.
// 2. Let audio language be a BCP 47 language tag for the language
// specified in the initialization segment for this track or an empty
// string if no language info is present.
// 3. If audio language equals an empty string or the 'und' BCP 47
// value, then run the default track language algorithm with
// byteStreamTrackID set to audio byte stream track ID and type set to
// "audio" and assign the value returned by the algorithm to audio
// language.
// 4. Let audio label be a label specified in the initialization segment
// for this track or an empty string if no label info is present.
// 5. If audio label equals an empty string, then run the default track
// label algorithm with byteStreamTrackID set to audio byte stream track
// ID and type set to "audio" and assign the value returned by the
// algorithm to audio label.
// 6. Let audio kinds be an array of kind strings specified in the
// initialization segment for this track or an empty array if no kind
// information is provided.
// 7. If audio kinds equals an empty array, then run the default track
// kinds algorithm with byteStreamTrackID set to audio byte stream track
// ID and type set to "audio" and assign the value returned by the
// algorithm to audio kinds.
// 8. For each value in audio kinds, run the following steps:
// 1. Let current audio kind equal the value from audio kinds for this
// iteration of the loop.
// 2. Let new audio track be a new AudioTrack object.
// 3. Generate a unique ID and assign it to the id property on new
// audio track.
// 4. Assign audio language to the language property on new audio
// track.
// 5. Assign audio label to the label property on new audio track.
// 6. Assign current audio kind to the kind property on new audio
// track.
// 7. If audioTracks.length equals 0, then run the following steps:
// 1. Set the enabled property on new audio track to true.
// 2. Set active track flag to true.
activeTrack = true;
// 8. Add new audio track to the audioTracks attribute on this
// SourceBuffer object.
// 9. Queue a task to fire a trusted event named addtrack, that does
// not bubble and is not cancelable, and that uses the TrackEvent
// interface, at the AudioTrackList object referenced by the
// audioTracks attribute on this SourceBuffer object.
// 10. Add new audio track to the audioTracks attribute on the
// HTMLMediaElement.
// 11. Queue a task to fire a trusted event named addtrack, that does
// not bubble and is not cancelable, and that uses the TrackEvent
// interface, at the AudioTrackList object referenced by the
// audioTracks attribute on the HTMLMediaElement.
mAudioTracks.mBuffers.AppendElement(TrackBuffer());
// 10. Add the track description for this track to the track buffer.
mAudioTracks.mInfo = new TrackInfoSharedPtr(info.mAudio, streamID);
mAudioTracks.mLastInfo = mAudioTracks.mInfo;
}
mVideoTracks.mNumTracks = numVideos;
// 3. For each video track in the initialization segment, run following
// steps: for (uint32_t i = 0; i < numVideos; i++) {
if (numVideos) {
// 1. Let video byte stream track ID be the Track ID for the current
// track being processed.
// 2. Let video language be a BCP 47 language tag for the language
// specified in the initialization segment for this track or an empty
// string if no language info is present.
// 3. If video language equals an empty string or the 'und' BCP 47
// value, then run the default track language algorithm with
// byteStreamTrackID set to video byte stream track ID and type set to
// "video" and assign the value returned by the algorithm to video
// language.
// 4. Let video label be a label specified in the initialization segment
// for this track or an empty string if no label info is present.
// 5. If video label equals an empty string, then run the default track
// label algorithm with byteStreamTrackID set to video byte stream track
// ID and type set to "video" and assign the value returned by the
// algorithm to video label.
// 6. Let video kinds be an array of kind strings specified in the
// initialization segment for this track or an empty array if no kind
// information is provided.
// 7. If video kinds equals an empty array, then run the default track
// kinds algorithm with byteStreamTrackID set to video byte stream track
// ID and type set to "video" and assign the value returned by the
// algorithm to video kinds.
// 8. For each value in video kinds, run the following steps:
// 1. Let current video kind equal the value from video kinds for this
// iteration of the loop.
// 2. Let new video track be a new VideoTrack object.
// 3. Generate a unique ID and assign it to the id property on new
// video track.
// 4. Assign video language to the language property on new video
// track.
// 5. Assign video label to the label property on new video track.
// 6. Assign current video kind to the kind property on new video
// track.
// 7. If videoTracks.length equals 0, then run the following steps:
// 1. Set the selected property on new video track to true.
// 2. Set active track flag to true.
activeTrack = true;
// 8. Add new video track to the videoTracks attribute on this
// SourceBuffer object.
// 9. Queue a task to fire a trusted event named addtrack, that does
// not bubble and is not cancelable, and that uses the TrackEvent
// interface, at the VideoTrackList object referenced by the
// videoTracks attribute on this SourceBuffer object.
// 10. Add new video track to the videoTracks attribute on the
// HTMLMediaElement.
// 11. Queue a task to fire a trusted event named addtrack, that does
// not bubble and is not cancelable, and that uses the TrackEvent
// interface, at the VideoTrackList object referenced by the
// videoTracks attribute on the HTMLMediaElement.
mVideoTracks.mBuffers.AppendElement(TrackBuffer());
// 10. Add the track description for this track to the track buffer.
mVideoTracks.mInfo = new TrackInfoSharedPtr(info.mVideo, streamID);
mVideoTracks.mLastInfo = mVideoTracks.mInfo;
}
// 4. For each text track in the initialization segment, run following
// steps:
// 5. If active track flag equals true, then run the following steps:
// This is handled by SourceBuffer once the promise is resolved.
if (activeTrack) {
mActiveTrack = true;
}
// 6. Set first initialization segment received flag to true.
mFirstInitializationSegmentReceived = true;
} else {
mAudioTracks.mLastInfo = new TrackInfoSharedPtr(info.mAudio, streamID);
mVideoTracks.mLastInfo = new TrackInfoSharedPtr(info.mVideo, streamID);
}
UniquePtr<EncryptionInfo> crypto = mInputDemuxer->GetCrypto();
if (crypto && crypto->IsEncrypted()) {
// Try and dispatch 'encrypted'. Won't go if ready state still
// HAVE_NOTHING.
for (uint32_t i = 0; i < crypto->mInitDatas.Length(); i++) {
nsCOMPtr<nsIRunnable> r = new DispatchKeyNeededEvent(
mParentDecoder, crypto->mInitDatas[i].mInitData,
crypto->mInitDatas[i].mType);
mAbstractMainThread->Dispatch(r.forget());
}
info.mCrypto = *crypto;
// We clear our crypto init data array, so the MediaFormatReader will
// not emit an encrypted event for the same init data again.
info.mCrypto.mInitDatas.Clear();
}
{
MutexAutoLock mut(mMutex);
mInfo = info;
}
// We now have a valid init data ; we can store it for later use.
mInitData = mParser->InitData();
}
// We have now completed the changeType operation.
mChangeTypeReceived = false;
// 3. Remove the initialization segment bytes from the beginning of the input
// buffer. This step has already been done in InitializationSegmentReceived
// when we transferred the content into mCurrentInputBuffer.
mCurrentInputBuffer->EvictAll();
mInputDemuxer->NotifyDataRemoved();
RecreateParser(true);
// 4. Set append state to WAITING_FOR_SEGMENT.
SetAppendState(AppendState::WAITING_FOR_SEGMENT);
// 5. Jump to the loop top step above.
ScheduleSegmentParserLoop();
if (aResult != NS_OK && mParentDecoder) {
RefPtr<TrackBuffersManager> self = this;
mAbstractMainThread->Dispatch(NS_NewRunnableFunction(
"TrackBuffersManager::OnDemuxerInitDone", [self, aResult]() {
if (self->mParentDecoder && self->mParentDecoder->GetOwner()) {
self->mParentDecoder->GetOwner()->DecodeWarning(aResult);
}
}));
}
}
void TrackBuffersManager::OnDemuxerInitFailed(const MediaResult& aError) {
MOZ_ASSERT(aError != NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA);
mDemuxerInitRequest.Complete();
RejectAppend(aError, __func__);
}
RefPtr<TrackBuffersManager::CodedFrameProcessingPromise>
TrackBuffersManager::CodedFrameProcessing() {
MOZ_ASSERT(OnTaskQueue());
MOZ_ASSERT(mProcessingPromise.IsEmpty());
MediaByteRange mediaRange = mParser->MediaSegmentRange();
if (mediaRange.IsEmpty()) {
AppendDataToCurrentInputBuffer(*mInputBuffer);
mInputBuffer.reset();
} else {
MOZ_ASSERT(mProcessedInput >= mInputBuffer->Length());
if (int64_t(mProcessedInput - mInputBuffer->Length()) > mediaRange.mEnd) {
// Something is not quite right with the data appended. Refuse it.
// This would typically happen if the previous media segment was partial
// yet a new complete media segment was added.
return CodedFrameProcessingPromise::CreateAndReject(NS_ERROR_FAILURE,
__func__);
}
// The mediaRange is offset by the init segment position previously added.
uint32_t length =
mediaRange.mEnd - (mProcessedInput - mInputBuffer->Length());
if (!length) {
// We've completed our earlier media segment and no new data is to be
// processed. This happens with some containers that can't detect that a
// media segment is ending until a new one starts.
RefPtr<CodedFrameProcessingPromise> p =
mProcessingPromise.Ensure(__func__);
CompleteCodedFrameProcessing();
return p;
}
AppendDataToCurrentInputBuffer(mInputBuffer->To(length));
mInputBuffer->RemoveFront(length);
}
RefPtr<CodedFrameProcessingPromise> p = mProcessingPromise.Ensure(__func__);
DoDemuxVideo();
return p;
}
void TrackBuffersManager::OnDemuxFailed(TrackType aTrack,
const MediaResult& aError) {
MOZ_ASSERT(OnTaskQueue());
MSE_DEBUG("Failed to demux %s, failure:%s",
aTrack == TrackType::kVideoTrack ? "video" : "audio",
aError.ErrorName().get());
switch (aError.Code()) {
case NS_ERROR_DOM_MEDIA_END_OF_STREAM:
case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA:
if (aTrack == TrackType::kVideoTrack) {
DoDemuxAudio();
} else {
CompleteCodedFrameProcessing();
}
break;
default:
RejectProcessing(aError, __func__);
break;
}
}
void TrackBuffersManager::DoDemuxVideo() {
MOZ_ASSERT(OnTaskQueue());
if (!HasVideo()) {
DoDemuxAudio();
return;
}
mVideoTracks.mDemuxer->GetSamples(-1)
->Then(TaskQueueFromTaskQueue(), __func__, this,
&TrackBuffersManager::OnVideoDemuxCompleted,
&TrackBuffersManager::OnVideoDemuxFailed)
->Track(mVideoTracks.mDemuxRequest);
}
void TrackBuffersManager::MaybeDispatchEncryptedEvent(
const nsTArray<RefPtr<MediaRawData>>& aSamples) {
// Try and dispatch 'encrypted'. Won't go if ready state still HAVE_NOTHING.
for (const RefPtr<MediaRawData>& sample : aSamples) {
for (const nsTArray<uint8_t>& initData : sample->mCrypto.mInitDatas) {
nsCOMPtr<nsIRunnable> r = new DispatchKeyNeededEvent(
mParentDecoder, initData, sample->mCrypto.mInitDataType);
mAbstractMainThread->Dispatch(r.forget());
}
}
}
void TrackBuffersManager::OnVideoDemuxCompleted(
RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
MOZ_ASSERT(OnTaskQueue());
MSE_DEBUG("%zu video samples demuxed", aSamples->GetSamples().Length());
mVideoTracks.mDemuxRequest.Complete();
mVideoTracks.mQueuedSamples.AppendElements(aSamples->GetSamples());
MaybeDispatchEncryptedEvent(aSamples->GetSamples());
DoDemuxAudio();
}
void TrackBuffersManager::DoDemuxAudio() {
MOZ_ASSERT(OnTaskQueue());
if (!HasAudio()) {
CompleteCodedFrameProcessing();
return;
}
mAudioTracks.mDemuxer->GetSamples(-1)
->Then(TaskQueueFromTaskQueue(), __func__, this,
&TrackBuffersManager::OnAudioDemuxCompleted,
&TrackBuffersManager::OnAudioDemuxFailed)
->Track(mAudioTracks.mDemuxRequest);
}
void TrackBuffersManager::OnAudioDemuxCompleted(
RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
MOZ_ASSERT(OnTaskQueue());
MSE_DEBUG("%zu audio samples demuxed", aSamples->GetSamples().Length());
mAudioTracks.mDemuxRequest.Complete();
mAudioTracks.mQueuedSamples.AppendElements(aSamples->GetSamples());
CompleteCodedFrameProcessing();
MaybeDispatchEncryptedEvent(aSamples->GetSamples());
}
void TrackBuffersManager::CompleteCodedFrameProcessing() {
MOZ_ASSERT(OnTaskQueue());
// 1. For each coded frame in the media segment run the following steps:
// Coded Frame Processing steps 1.1 to 1.21.
if (mSourceBufferAttributes->GetAppendMode() ==
SourceBufferAppendMode::Sequence &&
mVideoTracks.mQueuedSamples.Length() &&
mAudioTracks.mQueuedSamples.Length()) {
// When we are in sequence mode, the order in which we process the frames is
// important as it determines the future value of timestampOffset.
// So we process the earliest sample first. See bug 1293576.
TimeInterval videoInterval =
PresentationInterval(mVideoTracks.mQueuedSamples);
TimeInterval audioInterval =
PresentationInterval(mAudioTracks.mQueuedSamples);
if (audioInterval.mStart < videoInterval.mStart) {
ProcessFrames(mAudioTracks.mQueuedSamples, mAudioTracks);
ProcessFrames(mVideoTracks.mQueuedSamples, mVideoTracks);
} else {
ProcessFrames(mVideoTracks.mQueuedSamples, mVideoTracks);
ProcessFrames(mAudioTracks.mQueuedSamples, mAudioTracks);
}
} else {
ProcessFrames(mVideoTracks.mQueuedSamples, mVideoTracks);
ProcessFrames(mAudioTracks.mQueuedSamples, mAudioTracks);
}
#if defined(DEBUG)
if (HasVideo()) {
const auto& track = mVideoTracks.GetTrackBuffer();
MOZ_ASSERT(track.IsEmpty() || track[0]->mKeyframe);
for (uint32_t i = 1; i < track.Length(); i++) {
MOZ_ASSERT(
(track[i - 1]->mTrackInfo->GetID() == track[i]->mTrackInfo->GetID() &&
track[i - 1]->mTimecode <= track[i]->mTimecode) ||
track[i]->mKeyframe);
}
}
if (HasAudio()) {
const auto& track = mAudioTracks.GetTrackBuffer();
MOZ_ASSERT(track.IsEmpty() || track[0]->mKeyframe);
for (uint32_t i = 1; i < track.Length(); i++) {
MOZ_ASSERT(
(track[i - 1]->mTrackInfo->GetID() == track[i]->mTrackInfo->GetID() &&
track[i - 1]->mTimecode <= track[i]->mTimecode) ||
track[i]->mKeyframe);
}
}
#endif
mVideoTracks.mQueuedSamples.Clear();
mAudioTracks.mQueuedSamples.Clear();
UpdateBufferedRanges();
// Update our reported total size.
mSizeSourceBuffer = mVideoTracks.mSizeBuffer + mAudioTracks.mSizeBuffer;
// Return to step 6.4 of Segment Parser Loop algorithm
// 4. If this SourceBuffer is full and cannot accept more media data, then set
// the buffer full flag to true.
if (mSizeSourceBuffer >= EvictionThreshold()) {
mBufferFull = true;
}
// 5. If the input buffer does not contain a complete media segment, then jump
// to the need more data step below.
if (mParser->MediaSegmentRange().IsEmpty()) {
ResolveProcessing(true, __func__);
return;
}
mLastParsedEndTime = Some(std::max(mAudioTracks.mLastParsedEndTime,
mVideoTracks.mLastParsedEndTime));
// 6. Remove the media segment bytes from the beginning of the input buffer.
// Clear our demuxer from any already processed data.
int64_t safeToEvict =
std::min(HasVideo() ? mVideoTracks.mDemuxer->GetEvictionOffset(
mVideoTracks.mLastParsedEndTime)
: INT64_MAX,
HasAudio() ? mAudioTracks.mDemuxer->GetEvictionOffset(
mAudioTracks.mLastParsedEndTime)
: INT64_MAX);
mCurrentInputBuffer->EvictBefore(safeToEvict);
mInputDemuxer->NotifyDataRemoved();
RecreateParser(true);
// 7. Set append state to WAITING_FOR_SEGMENT.
SetAppendState(AppendState::WAITING_FOR_SEGMENT);
// 8. Jump to the loop top step above.
ResolveProcessing(false, __func__);
}
void TrackBuffersManager::RejectProcessing(const MediaResult& aRejectValue,
const char* aName) {
mProcessingPromise.RejectIfExists(aRejectValue, __func__);
}
void TrackBuffersManager::ResolveProcessing(bool aResolveValue,
const char* aName) {
mProcessingPromise.ResolveIfExists(aResolveValue, __func__);
}
void TrackBuffersManager::CheckSequenceDiscontinuity(
const TimeUnit& aPresentationTime) {
if (mSourceBufferAttributes->GetAppendMode() ==
SourceBufferAppendMode::Sequence &&
mSourceBufferAttributes->HaveGroupStartTimestamp()) {
mSourceBufferAttributes->SetTimestampOffset(
mSourceBufferAttributes->GetGroupStartTimestamp() - aPresentationTime);
mSourceBufferAttributes->SetGroupEndTimestamp(
mSourceBufferAttributes->GetGroupStartTimestamp());
mVideoTracks.mNeedRandomAccessPoint = true;
mAudioTracks.mNeedRandomAccessPoint = true;
mSourceBufferAttributes->ResetGroupStartTimestamp();
}
}
TimeInterval TrackBuffersManager::PresentationInterval(
const TrackBuffer& aSamples) const {
TimeInterval presentationInterval =
TimeInterval(aSamples[0]->mTime, aSamples[0]->GetEndTime());
for (uint32_t i = 1; i < aSamples.Length(); i++) {
auto& sample = aSamples[i];
presentationInterval = presentationInterval.Span(
TimeInterval(sample->mTime, sample->GetEndTime()));
}
return presentationInterval;
}
void TrackBuffersManager::ProcessFrames(TrackBuffer& aSamples,
TrackData& aTrackData) {
if (!aSamples.Length()) {
return;
}
// 1. If generate timestamps flag equals true
// Let presentation timestamp equal 0.
// Otherwise
// Let presentation timestamp be a double precision floating point
// representation of the coded frame's presentation timestamp in seconds.
TimeUnit presentationTimestamp = mSourceBufferAttributes->mGenerateTimestamps
? TimeUnit::Zero()
: aSamples[0]->mTime;
// 3. If mode equals "sequence" and group start timestamp is set, then run the
// following steps:
CheckSequenceDiscontinuity(presentationTimestamp);
// 5. Let track buffer equal the track buffer that the coded frame will be
// added to.
auto& trackBuffer = aTrackData;
TimeIntervals samplesRange;
uint32_t sizeNewSamples = 0;
TrackBuffer samples; // array that will contain the frames to be added
// to our track buffer.
// We assume that no frames are contiguous within a media segment and as such
// don't need to check for discontinuity except for the first frame and should
// a frame be ignored due to the target window.
bool needDiscontinuityCheck = true;
// Highest presentation time seen in samples block.
TimeUnit highestSampleTime;
if (aSamples.Length()) {
aTrackData.mLastParsedEndTime = TimeUnit();
}
auto addToSamples = [&](MediaRawData* aSample,
const TimeInterval& aInterval) {
aSample->mTime = aInterval.mStart;
aSample->mDuration = aInterval.Length();
aSample->mTrackInfo = trackBuffer.mLastInfo;
MOZ_DIAGNOSTIC_ASSERT(aSample->HasValidTime());
samplesRange += aInterval;
sizeNewSamples += aSample->ComputedSizeOfIncludingThis();
samples.AppendElement(aSample);
};
// Will be set to the last frame dropped due to being outside mAppendWindow.
// It will be added prior the first following frame which can be added to the
// track buffer.
// This sample will be set with a duration of only 1us which will cause it to
// be dropped once returned by the decoder.
// This sample is required to "prime" the decoder so that the following frame
// can be fully decoded.
RefPtr<MediaRawData> previouslyDroppedSample;
for (auto& sample : aSamples) {
const TimeUnit sampleEndTime = sample->GetEndTime();
if (sampleEndTime > aTrackData.mLastParsedEndTime) {
aTrackData.mLastParsedEndTime = sampleEndTime;
}
// We perform step 10 right away as we can't do anything should a keyframe
// be needed until we have one.
// 10. If the need random access point flag on track buffer equals true,
// then run the following steps:
if (trackBuffer.mNeedRandomAccessPoint) {
// 1. If the coded frame is not a random access point, then drop the coded
// frame and jump to the top of the loop to start processing the next
// coded frame.
if (!sample->mKeyframe) {
previouslyDroppedSample = nullptr;
continue;
}
// 2. Set the need random access point flag on track buffer to false.
trackBuffer.mNeedRandomAccessPoint = false;
}
// We perform step 1,2 and 4 at once:
// 1. If generate timestamps flag equals true:
// Let presentation timestamp equal 0.
// Let decode timestamp equal 0.
// Otherwise:
// Let presentation timestamp be a double precision floating point
// representation of the coded frame's presentation timestamp in seconds.
// Let decode timestamp be a double precision floating point
// representation of the coded frame's decode timestamp in seconds.
// 2. Let frame duration be a double precision floating point representation
// of the coded frame's duration in seconds. Step 3 is performed earlier or
// when a discontinuity has been detected.
// 4. If timestampOffset is not 0, then run the following steps:
TimeUnit sampleTime = sample->mTime;
TimeUnit sampleTimecode = sample->mTimecode;
TimeUnit sampleDuration = sample->mDuration;
TimeUnit timestampOffset = mSourceBufferAttributes->GetTimestampOffset();
TimeInterval sampleInterval =
mSourceBufferAttributes->mGenerateTimestamps
? TimeInterval(timestampOffset, timestampOffset + sampleDuration)
: TimeInterval(timestampOffset + sampleTime,
timestampOffset + sampleTime + sampleDuration);
TimeUnit decodeTimestamp = mSourceBufferAttributes->mGenerateTimestamps
? timestampOffset
: timestampOffset + sampleTimecode;
SAMPLE_DEBUG(
"Processing %s frame [%" PRId64 ",%" PRId64 "] (adjusted:[%" PRId64
",%" PRId64 "]), dts:%" PRId64 ", duration:%" PRId64 ", kf:%d)",
aTrackData.mInfo->mMimeType.get(), sample->mTime.ToMicroseconds(),
sample->GetEndTime().ToMicroseconds(),
sampleInterval.mStart.ToMicroseconds(),
sampleInterval.mEnd.ToMicroseconds(),
sample->mTimecode.ToMicroseconds(), sample->mDuration.ToMicroseconds(),
sample->mKeyframe);
// 6. If last decode timestamp for track buffer is set and decode timestamp
// is less than last decode timestamp: OR If last decode timestamp for track
// buffer is set and the difference between decode timestamp and last decode
// timestamp is greater than 2 times last frame duration:
if (needDiscontinuityCheck && trackBuffer.mLastDecodeTimestamp.isSome() &&
(decodeTimestamp < trackBuffer.mLastDecodeTimestamp.ref() ||
(decodeTimestamp - trackBuffer.mLastDecodeTimestamp.ref() >
trackBuffer.mLongestFrameDuration * 2))) {
MSE_DEBUG("Discontinuity detected.");
SourceBufferAppendMode appendMode =
mSourceBufferAttributes->GetAppendMode();
// 1a. If mode equals "segments":
if (appendMode == SourceBufferAppendMode::Segments) {
// Set group end timestamp to presentation timestamp.
mSourceBufferAttributes->SetGroupEndTimestamp(sampleInterval.mStart);
}
// 1b. If mode equals "sequence":
if (appendMode == SourceBufferAppendMode::Sequence) {
// Set group start timestamp equal to the group end timestamp.
mSourceBufferAttributes->SetGroupStartTimestamp(
mSourceBufferAttributes->GetGroupEndTimestamp());
}
for (auto& track : GetTracksList()) {
// 2. Unset the last decode timestamp on all track buffers.
// 3. Unset the last frame duration on all track buffers.
// 4. Unset the highest end timestamp on all track buffers.
// 5. Set the need random access point flag on all track buffers to
// true.
track->ResetAppendState();
}
// 6. Jump to the Loop Top step above to restart processing of the current
// coded frame. Rather that restarting the process for the frame, we run
// the first steps again instead.
// 3. If mode equals "sequence" and group start timestamp is set, then run
// the following steps:
TimeUnit presentationTimestamp =
mSourceBufferAttributes->mGenerateTimestamps ? TimeUnit()
: sampleTime;
CheckSequenceDiscontinuity(presentationTimestamp);
if (!sample->mKeyframe) {
previouslyDroppedSample = nullptr;
continue;
}
if (appendMode == SourceBufferAppendMode::Sequence) {
// mSourceBufferAttributes->GetTimestampOffset() was modified during
// CheckSequenceDiscontinuity. We need to update our variables.
timestampOffset = mSourceBufferAttributes->GetTimestampOffset();
sampleInterval =
mSourceBufferAttributes->mGenerateTimestamps
? TimeInterval(timestampOffset,
timestampOffset + sampleDuration)
: TimeInterval(timestampOffset + sampleTime,
timestampOffset + sampleTime + sampleDuration);
decodeTimestamp = mSourceBufferAttributes->mGenerateTimestamps
? timestampOffset
: timestampOffset + sampleTimecode;
}
trackBuffer.mNeedRandomAccessPoint = false;
needDiscontinuityCheck = false;
}
// 7. Let frame end timestamp equal the sum of presentation timestamp and
// frame duration. This is sampleInterval.mEnd
// 8. If presentation timestamp is less than appendWindowStart, then set the
// need random access point flag to true, drop the coded frame, and jump to
// the top of the loop to start processing the next coded frame.
// 9. If frame end timestamp is greater than appendWindowEnd, then set the
// need random access point flag to true, drop the coded frame, and jump to
// the top of the loop to start processing the next coded frame.
if (!mAppendWindow.ContainsStrict(sampleInterval)) {
if (mAppendWindow.IntersectsStrict(sampleInterval)) {
// 8. Note: Some implementations MAY choose to collect some of these
// coded frames with presentation timestamp less than
// appendWindowStart and use them to generate a splice at the first
// coded frame that has a presentation timestamp greater than or
// equal to appendWindowStart even if that frame is not a random
// access point. Supporting this requires multiple decoders or faster
// than real-time decoding so for now this behavior will not be a
// normative requirement.
// 9. Note: Some implementations MAY choose to collect coded frames with
// presentation timestamp less than appendWindowEnd and frame end
// timestamp greater than appendWindowEnd and use them to generate a
// splice across the portion of the collected coded frames within the
// append window at time of collection, and the beginning portion of
// later processed frames which only partially overlap the end of the
// collected coded frames. Supporting this requires multiple decoders
// or faster than real-time decoding so for now this behavior will
// not be a normative requirement. In conjunction with collecting
// coded frames that span appendWindowStart, implementations MAY thus
// support gapless audio splicing.
TimeInterval intersection = mAppendWindow.