Source code

Revision control

Other Tools

1
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
3
/* This Source Code Form is subject to the terms of the Mozilla Public
4
* License, v. 2.0. If a copy of the MPL was not distributed with this
5
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7
#include "TrackBuffersManager.h"
8
#include "ContainerParser.h"
9
#include "MediaSourceDemuxer.h"
10
#include "MediaSourceUtils.h"
11
#include "SourceBuffer.h"
12
#include "SourceBufferResource.h"
13
#include "SourceBufferTask.h"
14
#include "WebMDemuxer.h"
15
#include "mozilla/ErrorResult.h"
16
#include "mozilla/Preferences.h"
17
#include "mozilla/StaticPrefs_media.h"
18
#include "nsMimeTypes.h"
19
20
#ifdef MOZ_FMP4
21
# include "MP4Demuxer.h"
22
#endif
23
24
#include <limits>
25
26
extern mozilla::LogModule* GetMediaSourceLog();
27
28
#define MSE_DEBUG(arg, ...) \
29
DDMOZ_LOG(GetMediaSourceLog(), mozilla::LogLevel::Debug, "(%s)::%s: " arg, \
30
mType.OriginalString().Data(), __func__, ##__VA_ARGS__)
31
#define MSE_DEBUGV(arg, ...) \
32
DDMOZ_LOG(GetMediaSourceLog(), mozilla::LogLevel::Verbose, "(%s)::%s: " arg, \
33
mType.OriginalString().Data(), __func__, ##__VA_ARGS__)
34
35
mozilla::LogModule* GetMediaSourceSamplesLog() {
36
static mozilla::LazyLogModule sLogModule("MediaSourceSamples");
37
return sLogModule;
38
}
39
#define SAMPLE_DEBUG(arg, ...) \
40
DDMOZ_LOG(GetMediaSourceSamplesLog(), mozilla::LogLevel::Debug, \
41
"(%s)::%s: " arg, mType.OriginalString().Data(), __func__, \
42
##__VA_ARGS__)
43
44
namespace mozilla {
45
46
using dom::SourceBufferAppendMode;
47
using media::TimeInterval;
48
using media::TimeIntervals;
49
using media::TimeUnit;
50
typedef SourceBufferTask::AppendBufferResult AppendBufferResult;
51
typedef SourceBufferAttributes::AppendState AppendState;
52
53
static const char* AppendStateToStr(AppendState aState) {
54
switch (aState) {
55
case AppendState::WAITING_FOR_SEGMENT:
56
return "WAITING_FOR_SEGMENT";
57
case AppendState::PARSING_INIT_SEGMENT:
58
return "PARSING_INIT_SEGMENT";
59
case AppendState::PARSING_MEDIA_SEGMENT:
60
return "PARSING_MEDIA_SEGMENT";
61
default:
62
return "IMPOSSIBLE";
63
}
64
}
65
66
static Atomic<uint32_t> sStreamSourceID(0u);
67
68
class DispatchKeyNeededEvent : public Runnable {
69
public:
70
DispatchKeyNeededEvent(MediaSourceDecoder* aDecoder,
71
const nsTArray<uint8_t>& aInitData,
72
const nsString& aInitDataType)
73
: Runnable("DispatchKeyNeededEvent"),
74
mDecoder(aDecoder),
75
mInitData(aInitData),
76
mInitDataType(aInitDataType) {}
77
NS_IMETHOD Run() override {
78
// Note: Null check the owner, as the decoder could have been shutdown
79
// since this event was dispatched.
80
MediaDecoderOwner* owner = mDecoder->GetOwner();
81
if (owner) {
82
owner->DispatchEncrypted(mInitData, mInitDataType);
83
}
84
mDecoder = nullptr;
85
return NS_OK;
86
}
87
88
private:
89
RefPtr<MediaSourceDecoder> mDecoder;
90
nsTArray<uint8_t> mInitData;
91
nsString mInitDataType;
92
};
93
94
TrackBuffersManager::TrackBuffersManager(MediaSourceDecoder* aParentDecoder,
95
const MediaContainerType& aType)
96
: mBufferFull(false),
97
mFirstInitializationSegmentReceived(false),
98
mChangeTypeReceived(false),
99
mNewMediaSegmentStarted(false),
100
mActiveTrack(false),
101
mType(aType),
102
mParser(ContainerParser::CreateForMIMEType(aType)),
103
mProcessedInput(0),
104
mParentDecoder(new nsMainThreadPtrHolder<MediaSourceDecoder>(
105
"TrackBuffersManager::mParentDecoder", aParentDecoder,
106
false /* strict */)),
107
mAbstractMainThread(aParentDecoder->AbstractMainThread()),
108
mEnded(false),
109
mVideoEvictionThreshold(Preferences::GetUint(
110
"media.mediasource.eviction_threshold.video", 100 * 1024 * 1024)),
111
mAudioEvictionThreshold(Preferences::GetUint(
112
"media.mediasource.eviction_threshold.audio", 20 * 1024 * 1024)),
113
mEvictionState(EvictionState::NO_EVICTION_NEEDED),
114
mMutex("TrackBuffersManager"),
115
mTaskQueue(aParentDecoder->GetDemuxer()->GetTaskQueue()) {
116
MOZ_ASSERT(NS_IsMainThread(), "Must be instanciated on the main thread");
117
DDLINKCHILD("parser", mParser.get());
118
}
119
120
TrackBuffersManager::~TrackBuffersManager() { ShutdownDemuxers(); }
121
122
RefPtr<TrackBuffersManager::AppendPromise> TrackBuffersManager::AppendData(
123
already_AddRefed<MediaByteBuffer> aData,
124
const SourceBufferAttributes& aAttributes) {
125
MOZ_ASSERT(NS_IsMainThread());
126
RefPtr<MediaByteBuffer> data(aData);
127
MSE_DEBUG("Appending %zu bytes", data->Length());
128
129
mEnded = false;
130
131
return InvokeAsync(static_cast<AbstractThread*>(GetTaskQueueSafe().get()),
132
this, __func__, &TrackBuffersManager::DoAppendData,
133
data.forget(), aAttributes);
134
}
135
136
RefPtr<TrackBuffersManager::AppendPromise> TrackBuffersManager::DoAppendData(
137
already_AddRefed<MediaByteBuffer> aData,
138
const SourceBufferAttributes& aAttributes) {
139
RefPtr<AppendBufferTask> task =
140
new AppendBufferTask(std::move(aData), aAttributes);
141
RefPtr<AppendPromise> p = task->mPromise.Ensure(__func__);
142
QueueTask(task);
143
144
return p;
145
}
146
147
void TrackBuffersManager::QueueTask(SourceBufferTask* aTask) {
148
// The source buffer is a wrapped native, it would be unlinked twice and so
149
// the TrackBuffersManager::Detach() would also be called twice. Since the
150
// detach task has been done before, we could ignore this task.
151
RefPtr<TaskQueue> taskQueue = GetTaskQueueSafe();
152
if (!taskQueue) {
153
MOZ_ASSERT(aTask->GetType() == SourceBufferTask::Type::Detach,
154
"only detach task could happen here!");
155
MSE_DEBUG("Could not queue the task '%s' without task queue",
156
aTask->GetTypeName());
157
return;
158
}
159
160
if (!taskQueue->IsCurrentThreadIn()) {
161
nsresult rv =
162
taskQueue->Dispatch(NewRunnableMethod<RefPtr<SourceBufferTask>>(
163
"TrackBuffersManager::QueueTask", this,
164
&TrackBuffersManager::QueueTask, aTask));
165
MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
166
Unused << rv;
167
return;
168
}
169
mQueue.Push(aTask);
170
ProcessTasks();
171
}
172
173
void TrackBuffersManager::ProcessTasks() {
174
// ProcessTask is always called OnTaskQueue, however it is possible that it is
175
// called once again after a first Detach task has run, in which case
176
// mTaskQueue would be null.
177
// This can happen under two conditions:
178
// 1- Two Detach tasks were queued in a row due to a double cycle collection.
179
// 2- An call to ProcessTasks() had queued another run of ProcessTasks while
180
// a Detach task is pending.
181
// We handle these two cases by aborting early.
182
// A second Detach task was queued, prior the first one running, ignore it.
183
if (!mTaskQueue) {
184
RefPtr<SourceBufferTask> task = mQueue.Pop();
185
if (!task) {
186
return;
187
}
188
MOZ_RELEASE_ASSERT(task->GetType() == SourceBufferTask::Type::Detach,
189
"only detach task could happen here!");
190
MSE_DEBUG("Could not process the task '%s' after detached",
191
task->GetTypeName());
192
return;
193
}
194
195
MOZ_ASSERT(OnTaskQueue());
196
typedef SourceBufferTask::Type Type;
197
198
if (mCurrentTask) {
199
// Already have a task pending. ProcessTask will be scheduled once the
200
// current task complete.
201
return;
202
}
203
RefPtr<SourceBufferTask> task = mQueue.Pop();
204
if (!task) {
205
// nothing to do.
206
return;
207
}
208
209
MSE_DEBUG("Process task '%s'", task->GetTypeName());
210
switch (task->GetType()) {
211
case Type::AppendBuffer:
212
mCurrentTask = task;
213
if (!mInputBuffer || mInputBuffer->IsEmpty()) {
214
// Note: we reset mInputBuffer here to ensure it doesn't grow unbounded.
215
mInputBuffer.reset();
216
mInputBuffer = Some(MediaSpan(task->As<AppendBufferTask>()->mBuffer));
217
} else if (!mInputBuffer->Append(task->As<AppendBufferTask>()->mBuffer)) {
218
RejectAppend(NS_ERROR_OUT_OF_MEMORY, __func__);
219
return;
220
}
221
mSourceBufferAttributes = MakeUnique<SourceBufferAttributes>(
222
task->As<AppendBufferTask>()->mAttributes);
223
mAppendWindow = TimeInterval(
224
TimeUnit::FromSeconds(
225
mSourceBufferAttributes->GetAppendWindowStart()),
226
TimeUnit::FromSeconds(mSourceBufferAttributes->GetAppendWindowEnd()));
227
ScheduleSegmentParserLoop();
228
break;
229
case Type::RangeRemoval: {
230
bool rv = CodedFrameRemoval(task->As<RangeRemovalTask>()->mRange);
231
task->As<RangeRemovalTask>()->mPromise.Resolve(rv, __func__);
232
break;
233
}
234
case Type::EvictData:
235
DoEvictData(task->As<EvictDataTask>()->mPlaybackTime,
236
task->As<EvictDataTask>()->mSizeToEvict);
237
break;
238
case Type::Abort:
239
// not handled yet, and probably never.
240
break;
241
case Type::Reset:
242
CompleteResetParserState();
243
break;
244
case Type::Detach:
245
mCurrentInputBuffer = nullptr;
246
MOZ_DIAGNOSTIC_ASSERT(mQueue.Length() == 0,
247
"Detach task must be the last");
248
mVideoTracks.Reset();
249
mAudioTracks.Reset();
250
ShutdownDemuxers();
251
ResetTaskQueue();
252
return;
253
case Type::ChangeType:
254
MOZ_RELEASE_ASSERT(!mCurrentTask);
255
mType = task->As<ChangeTypeTask>()->mType;
256
mChangeTypeReceived = true;
257
mInitData = nullptr;
258
// A new input buffer will be created once we receive a new init segment.
259
// The first segment received after a changeType call must be an init
260
// segment.
261
mCurrentInputBuffer = nullptr;
262
CompleteResetParserState();
263
break;
264
default:
265
NS_WARNING("Invalid Task");
266
}
267
TaskQueueFromTaskQueue()->Dispatch(
268
NewRunnableMethod("TrackBuffersManager::ProcessTasks", this,
269
&TrackBuffersManager::ProcessTasks));
270
}
271
272
// The MSE spec requires that we abort the current SegmentParserLoop
273
// which is then followed by a call to ResetParserState.
274
// However due to our asynchronous design this causes inherent difficulties.
275
// As the spec behaviour is non deterministic anyway, we instead process all
276
// pending frames found in the input buffer.
277
void TrackBuffersManager::AbortAppendData() {
278
MOZ_ASSERT(NS_IsMainThread());
279
MSE_DEBUG("");
280
281
QueueTask(new AbortTask());
282
}
283
284
void TrackBuffersManager::ResetParserState(
285
SourceBufferAttributes& aAttributes) {
286
MOZ_ASSERT(NS_IsMainThread());
287
MSE_DEBUG("");
288
289
// Spec states:
290
// 1. If the append state equals PARSING_MEDIA_SEGMENT and the input buffer
291
// contains some complete coded frames, then run the coded frame processing
292
// algorithm until all of these complete coded frames have been processed.
293
// However, we will wait until all coded frames have been processed regardless
294
// of the value of append state.
295
QueueTask(new ResetTask());
296
297
// ResetParserState has some synchronous steps that much be performed now.
298
// The remaining steps will be performed once the ResetTask gets executed.
299
300
// 6. If the mode attribute equals "sequence", then set the group start
301
// timestamp to the group end timestamp
302
if (aAttributes.GetAppendMode() == SourceBufferAppendMode::Sequence) {
303
aAttributes.SetGroupStartTimestamp(aAttributes.GetGroupEndTimestamp());
304
}
305
// 8. Set append state to WAITING_FOR_SEGMENT.
306
aAttributes.SetAppendState(AppendState::WAITING_FOR_SEGMENT);
307
}
308
309
RefPtr<TrackBuffersManager::RangeRemovalPromise>
310
TrackBuffersManager::RangeRemoval(TimeUnit aStart, TimeUnit aEnd) {
311
MOZ_ASSERT(NS_IsMainThread());
312
MSE_DEBUG("From %.2f to %.2f", aStart.ToSeconds(), aEnd.ToSeconds());
313
314
mEnded = false;
315
316
return InvokeAsync(static_cast<AbstractThread*>(GetTaskQueueSafe().get()),
317
this, __func__,
318
&TrackBuffersManager::CodedFrameRemovalWithPromise,
319
TimeInterval(aStart, aEnd));
320
}
321
322
TrackBuffersManager::EvictDataResult TrackBuffersManager::EvictData(
323
const TimeUnit& aPlaybackTime, int64_t aSize) {
324
MOZ_ASSERT(NS_IsMainThread());
325
326
if (aSize > EvictionThreshold()) {
327
// We're adding more data than we can hold.
328
return EvictDataResult::BUFFER_FULL;
329
}
330
const int64_t toEvict = GetSize() + aSize - EvictionThreshold();
331
332
const uint32_t canEvict =
333
Evictable(HasVideo() ? TrackInfo::kVideoTrack : TrackInfo::kAudioTrack);
334
335
MSE_DEBUG("currentTime=%" PRId64 " buffered=%" PRId64
336
"kB, eviction threshold=%" PRId64
337
"kB, "
338
"evict=%" PRId64 "kB canevict=%" PRIu32 "kB",
339
aPlaybackTime.ToMicroseconds(), GetSize() / 1024,
340
EvictionThreshold() / 1024, toEvict / 1024, canEvict / 1024);
341
342
if (toEvict <= 0) {
343
mEvictionState = EvictionState::NO_EVICTION_NEEDED;
344
return EvictDataResult::NO_DATA_EVICTED;
345
}
346
347
EvictDataResult result;
348
349
if (mBufferFull && mEvictionState == EvictionState::EVICTION_COMPLETED &&
350
canEvict < uint32_t(toEvict)) {
351
// Our buffer is currently full. We will make another eviction attempt.
352
// However, the current appendBuffer will fail as we can't know ahead of
353
// time if the eviction will later succeed.
354
result = EvictDataResult::BUFFER_FULL;
355
} else {
356
mEvictionState = EvictionState::EVICTION_NEEDED;
357
result = EvictDataResult::NO_DATA_EVICTED;
358
}
359
MSE_DEBUG("Reached our size limit, schedule eviction of %" PRId64
360
" bytes (%s)",
361
toEvict,
362
result == EvictDataResult::BUFFER_FULL ? "buffer full"
363
: "no data evicted");
364
QueueTask(new EvictDataTask(aPlaybackTime, toEvict));
365
366
return result;
367
}
368
369
void TrackBuffersManager::ChangeType(const MediaContainerType& aType) {
370
MOZ_ASSERT(NS_IsMainThread());
371
372
QueueTask(new ChangeTypeTask(aType));
373
}
374
375
TimeIntervals TrackBuffersManager::Buffered() const {
376
MSE_DEBUG("");
377
379
380
MutexAutoLock mut(mMutex);
381
nsTArray<const TimeIntervals*> tracks;
382
if (HasVideo()) {
383
tracks.AppendElement(&mVideoBufferedRanges);
384
}
385
if (HasAudio()) {
386
tracks.AppendElement(&mAudioBufferedRanges);
387
}
388
389
// 2. Let highest end time be the largest track buffer ranges end time across
390
// all the track buffers managed by this SourceBuffer object.
391
TimeUnit highestEndTime = HighestEndTime(tracks);
392
393
// 3. Let intersection ranges equal a TimeRange object containing a single
394
// range from 0 to highest end time.
395
TimeIntervals intersection{
396
TimeInterval(TimeUnit::FromSeconds(0), highestEndTime)};
397
398
// 4. For each track buffer managed by this SourceBuffer, run the following
399
// steps:
400
// 1. Let track ranges equal the track buffer ranges for the current track
401
// buffer.
402
for (const TimeIntervals* trackRanges : tracks) {
403
// 2. If readyState is "ended", then set the end time on the last range in
404
// track ranges to highest end time.
405
// 3. Let new intersection ranges equal the intersection between the
406
// intersection ranges and the track ranges.
407
if (mEnded) {
408
TimeIntervals tR = *trackRanges;
409
tR.Add(TimeInterval(tR.GetEnd(), highestEndTime));
410
intersection.Intersection(tR);
411
} else {
412
intersection.Intersection(*trackRanges);
413
}
414
}
415
return intersection;
416
}
417
418
int64_t TrackBuffersManager::GetSize() const { return mSizeSourceBuffer; }
419
420
void TrackBuffersManager::Ended() { mEnded = true; }
421
422
void TrackBuffersManager::Detach() {
423
MOZ_ASSERT(NS_IsMainThread());
424
MSE_DEBUG("");
425
QueueTask(new DetachTask());
426
}
427
428
void TrackBuffersManager::CompleteResetParserState() {
429
MOZ_ASSERT(OnTaskQueue());
430
MSE_DEBUG("");
431
432
// We shouldn't change mInputDemuxer while a demuxer init/reset request is
433
// being processed. See bug 1239983.
434
MOZ_DIAGNOSTIC_ASSERT(!mDemuxerInitRequest.Exists(),
435
"Previous AppendBuffer didn't complete");
436
437
for (auto& track : GetTracksList()) {
438
// 2. Unset the last decode timestamp on all track buffers.
439
// 3. Unset the last frame duration on all track buffers.
440
// 4. Unset the highest end timestamp on all track buffers.
441
// 5. Set the need random access point flag on all track buffers to true.
442
track->ResetAppendState();
443
444
// if we have been aborted, we may have pending frames that we are going
445
// to discard now.
446
track->mQueuedSamples.Clear();
447
}
448
449
// 7. Remove all bytes from the input buffer.
450
mPendingInputBuffer.reset();
451
mInputBuffer.reset();
452
if (mCurrentInputBuffer) {
453
mCurrentInputBuffer->EvictAll();
454
// The demuxer will be recreated during the next run of SegmentParserLoop.
455
// As such we don't need to notify it that data has been removed.
456
mCurrentInputBuffer = new SourceBufferResource();
457
}
458
459
// We could be left with a demuxer in an unusable state. It needs to be
460
// recreated. Unless we have a pending changeType operation, we store in the
461
// InputBuffer an init segment which will be parsed during the next Segment
462
// Parser Loop and a new demuxer will be created and initialized.
463
// If we are in the middle of a changeType operation, then we do not have an
464
// init segment yet. The next appendBuffer operation will need to provide such
465
// init segment.
466
if (mFirstInitializationSegmentReceived && !mChangeTypeReceived) {
467
MOZ_ASSERT(mInitData && mInitData->Length(),
468
"we must have an init segment");
469
// The aim here is really to destroy our current demuxer.
470
CreateDemuxerforMIMEType();
471
// Recreate our input buffer. We can't directly assign the initData buffer
472
// to mInputBuffer as it will get modified in the Segment Parser Loop.
473
mInputBuffer = Some(MediaSpan::WithCopyOf(mInitData));
474
RecreateParser(true);
475
} else {
476
RecreateParser(false);
477
}
478
}
479
480
int64_t TrackBuffersManager::EvictionThreshold() const {
481
if (HasVideo()) {
482
return mVideoEvictionThreshold;
483
}
484
return mAudioEvictionThreshold;
485
}
486
487
void TrackBuffersManager::DoEvictData(const TimeUnit& aPlaybackTime,
488
int64_t aSizeToEvict) {
489
MOZ_ASSERT(OnTaskQueue());
490
491
mEvictionState = EvictionState::EVICTION_COMPLETED;
492
493
// Video is what takes the most space, only evict there if we have video.
494
auto& track = HasVideo() ? mVideoTracks : mAudioTracks;
495
const auto& buffer = track.GetTrackBuffer();
496
if (buffer.IsEmpty()) {
497
// Buffer has been emptied while the eviction was queued, nothing to do.
498
return;
499
}
500
if (track.mBufferedRanges.IsEmpty()) {
501
MSE_DEBUG(
502
"DoEvictData running with no buffered ranges. 0 duration data likely "
503
"present in our buffer(s). Evicting all data!");
504
// We have no buffered ranges, but may still have data. This happens if the
505
// buffer is full of 0 duration data. Normal removal procedures don't clear
506
// 0 duration data, so blow away all our data.
507
RemoveAllCodedFrames();
508
return;
509
}
510
// Remove any data we've already played, or before the next sample to be
511
// demuxed whichever is lowest.
512
TimeUnit lowerLimit = std::min(track.mNextSampleTime, aPlaybackTime);
513
uint32_t lastKeyFrameIndex = 0;
514
int64_t toEvict = aSizeToEvict;
515
int64_t partialEvict = 0;
516
for (uint32_t i = 0; i < buffer.Length(); i++) {
517
const auto& frame = buffer[i];
518
if (frame->mKeyframe) {
519
lastKeyFrameIndex = i;
520
toEvict -= partialEvict;
521
if (toEvict < 0) {
522
break;
523
}
524
partialEvict = 0;
525
}
526
if (frame->GetEndTime() >= lowerLimit) {
527
break;
528
}
529
partialEvict += frame->ComputedSizeOfIncludingThis();
530
}
531
532
const int64_t finalSize = mSizeSourceBuffer - aSizeToEvict;
533
534
if (lastKeyFrameIndex > 0) {
535
MSE_DEBUG("Step1. Evicting %" PRId64 " bytes prior currentTime",
536
aSizeToEvict - toEvict);
537
TimeUnit start = track.mBufferedRanges[0].mStart;
538
TimeUnit end =
539
buffer[lastKeyFrameIndex]->mTime - TimeUnit::FromMicroseconds(1);
540
if (end > start) {
541
CodedFrameRemoval(TimeInterval(start, end));
542
}
543
}
544
545
if (mSizeSourceBuffer <= finalSize) {
546
return;
547
}
548
549
toEvict = mSizeSourceBuffer - finalSize;
550
551
// See if we can evict data into the future.
552
// We do not evict data from the currently used buffered interval.
553
554
TimeUnit currentPosition = std::max(aPlaybackTime, track.mNextSampleTime);
555
TimeIntervals futureBuffered(
556
TimeInterval(currentPosition, TimeUnit::FromInfinity()));
557
futureBuffered.Intersection(track.mBufferedRanges);
558
futureBuffered.SetFuzz(MediaSourceDemuxer::EOS_FUZZ / 2);
559
if (futureBuffered.Length() <= 1) {
560
// We have one continuous segment ahead of us:
561
// nothing further can be evicted.
562
return;
563
}
564
565
// Don't evict before the end of the current segment
566
TimeUnit upperLimit = futureBuffered[0].mEnd;
567
uint32_t evictedFramesStartIndex = buffer.Length();
568
for (int32_t i = buffer.Length() - 1; i >= 0; i--) {
569
const auto& frame = buffer[i];
570
if (frame->mTime <= upperLimit || toEvict < 0) {
571
// We've reached a frame that shouldn't be evicted -> Evict after it ->
572
// i+1. Or the previous loop reached the eviction threshold -> Evict from
573
// it -> i+1.
574
evictedFramesStartIndex = i + 1;
575
break;
576
}
577
toEvict -= frame->ComputedSizeOfIncludingThis();
578
}
579
if (evictedFramesStartIndex < buffer.Length()) {
580
MSE_DEBUG("Step2. Evicting %" PRId64 " bytes from trailing data",
581
mSizeSourceBuffer - finalSize - toEvict);
582
CodedFrameRemoval(TimeInterval(buffer[evictedFramesStartIndex]->mTime,
583
TimeUnit::FromInfinity()));
584
}
585
}
586
587
RefPtr<TrackBuffersManager::RangeRemovalPromise>
588
TrackBuffersManager::CodedFrameRemovalWithPromise(TimeInterval aInterval) {
589
MOZ_ASSERT(OnTaskQueue());
590
591
RefPtr<RangeRemovalTask> task = new RangeRemovalTask(aInterval);
592
RefPtr<RangeRemovalPromise> p = task->mPromise.Ensure(__func__);
593
QueueTask(task);
594
595
return p;
596
}
597
598
bool TrackBuffersManager::CodedFrameRemoval(TimeInterval aInterval) {
599
MOZ_ASSERT(OnTaskQueue());
600
MSE_DEBUG("From %.2fs to %.2f", aInterval.mStart.ToSeconds(),
601
aInterval.mEnd.ToSeconds());
602
603
#if DEBUG
604
if (HasVideo()) {
605
MSE_DEBUG("before video ranges=%s",
606
DumpTimeRanges(mVideoTracks.mBufferedRanges).get());
607
}
608
if (HasAudio()) {
609
MSE_DEBUG("before audio ranges=%s",
610
DumpTimeRanges(mAudioTracks.mBufferedRanges).get());
611
}
612
#endif
613
614
// 1. Let start be the starting presentation timestamp for the removal range.
615
TimeUnit start = aInterval.mStart;
616
// 2. Let end be the end presentation timestamp for the removal range.
617
TimeUnit end = aInterval.mEnd;
618
619
bool dataRemoved = false;
620
621
// 3. For each track buffer in this source buffer, run the following steps:
622
for (auto track : GetTracksList()) {
623
MSE_DEBUGV("Processing %s track", track->mInfo->mMimeType.get());
624
// 1. Let remove end timestamp be the current value of duration
626
// At worse we will remove all frames until the end, unless a key frame is
627
// found between the current interval's end and the trackbuffer's end.
628
TimeUnit removeEndTimestamp = track->mBufferedRanges.GetEnd();
629
630
if (start > removeEndTimestamp) {
631
// Nothing to remove.
632
continue;
633
}
634
635
// 2. If this track buffer has a random access point timestamp that is
636
// greater than or equal to end, then update remove end timestamp to that
637
// random access point timestamp.
638
if (end < track->mBufferedRanges.GetEnd()) {
639
for (auto& frame : track->GetTrackBuffer()) {
640
if (frame->mKeyframe && frame->mTime >= end) {
641
removeEndTimestamp = frame->mTime;
642
break;
643
}
644
}
645
}
646
647
// 3. Remove all media data, from this track buffer, that contain starting
648
// timestamps greater than or equal to start and less than the remove end
649
// timestamp.
650
// 4. Remove decoding dependencies of the coded frames removed in the
651
// previous step: Remove all coded frames between the coded frames removed
652
// in the previous step and the next random access point after those removed
653
// frames.
654
TimeIntervals removedInterval{TimeInterval(start, removeEndTimestamp)};
655
RemoveFrames(removedInterval, *track, 0, RemovalMode::kRemoveFrame);
656
657
// 5. If this object is in activeSourceBuffers, the current playback
658
// position is greater than or equal to start and less than the remove end
659
// timestamp, and HTMLMediaElement.readyState is greater than HAVE_METADATA,
660
// then set the HTMLMediaElement.readyState attribute to HAVE_METADATA and
661
// stall playback. This will be done by the MDSM during playback.
662
// TODO properly, so it works even if paused.
663
}
664
665
UpdateBufferedRanges();
666
667
// Update our reported total size.
668
mSizeSourceBuffer = mVideoTracks.mSizeBuffer + mAudioTracks.mSizeBuffer;
669
670
// 4. If buffer full flag equals true and this object is ready to accept more
671
// bytes, then set the buffer full flag to false.
672
if (mBufferFull && mSizeSourceBuffer < EvictionThreshold()) {
673
mBufferFull = false;
674
}
675
676
return dataRemoved;
677
}
678
679
void TrackBuffersManager::RemoveAllCodedFrames() {
680
// This is similar to RemoveCodedFrames, but will attempt to remove ALL
681
// the frames. This is not to spec, as explained below at step 3.1. Steps
682
// below coincide with Remove Coded Frames algorithm from the spec.
683
MSE_DEBUG("RemoveAllCodedFrames called.");
684
MOZ_ASSERT(OnTaskQueue());
685
686
// 1. Let start be the starting presentation timestamp for the removal range.
687
TimeUnit start{};
688
// 2. Let end be the end presentation timestamp for the removal range.
689
TimeUnit end = TimeUnit::FromMicroseconds(1);
690
// Find an end time such that our range will include every frame in every
691
// track. We do this by setting the end of our interval to the largest end
692
// time seen + 1 microsecond.
693
for (TrackData* track : GetTracksList()) {
694
for (auto& frame : track->GetTrackBuffer()) {
695
MOZ_ASSERT(frame->mTime >= start,
696
"Shouldn't have frame at negative time!");
697
TimeUnit frameEnd = frame->mTime + frame->mDuration;
698
if (frameEnd > end) {
699
end = frameEnd + TimeUnit::FromMicroseconds(1);
700
}
701
}
702
}
703
704
// 3. For each track buffer in this source buffer, run the following steps:
705
TimeIntervals removedInterval{TimeInterval(start, end)};
706
for (TrackData* track : GetTracksList()) {
707
// 1. Let remove end timestamp be the current value of duration
708
// ^ It's off spec, but we ignore this in order to clear 0 duration frames.
709
// If we don't ignore this rule and our buffer is full of 0 duration frames
710
// at timestamp n, we get an eviction range of [0, n). When we get to step
711
// 3.3 below, the 0 duration frames will not be evicted because their
712
// timestamp is not less than remove end timestamp -- it will in fact be
713
// equal to remove end timestamp.
714
//
715
// 2. If this track buffer has a random access point timestamp that is
716
// greater than or equal to end, then update remove end timestamp to that
717
// random access point timestamp.
718
// ^ We've made sure end > any sample's timestamp, so can skip this.
719
//
720
// 3. Remove all media data, from this track buffer, that contain starting
721
// timestamps greater than or equal to start and less than the remove end
722
// timestamp.
723
// 4. Remove decoding dependencies of the coded frames removed in the
724
// previous step: Remove all coded frames between the coded frames removed
725
// in the previous step and the next random access point after those removed
726
// frames.
727
728
// This should remove every frame in the track because removedInterval was
729
// constructed such that every frame in any track falls into that interval.
730
RemoveFrames(removedInterval, *track, 0, RemovalMode::kRemoveFrame);
731
732
// 5. If this object is in activeSourceBuffers, the current playback
733
// position is greater than or equal to start and less than the remove end
734
// timestamp, and HTMLMediaElement.readyState is greater than HAVE_METADATA,
735
// then set the HTMLMediaElement.readyState attribute to HAVE_METADATA and
736
// stall playback. This will be done by the MDSM during playback.
737
// TODO properly, so it works even if paused.
738
}
739
740
UpdateBufferedRanges();
741
MOZ_ASSERT(mAudioBufferedRanges.IsEmpty(),
742
"Should have no buffered video ranges after evicting everything.");
743
MOZ_ASSERT(mVideoBufferedRanges.IsEmpty(),
744
"Should have no buffered video ranges after evicting everything.");
745
mSizeSourceBuffer = mVideoTracks.mSizeBuffer + mAudioTracks.mSizeBuffer;
746
MOZ_ASSERT(mSizeSourceBuffer == 0,
747
"Buffer should be empty after evicting everything!");
748
if (mBufferFull && mSizeSourceBuffer < EvictionThreshold()) {
749
mBufferFull = false;
750
}
751
}
752
753
void TrackBuffersManager::UpdateBufferedRanges() {
754
MutexAutoLock mut(mMutex);
755
756
mVideoBufferedRanges = mVideoTracks.mSanitizedBufferedRanges;
757
mAudioBufferedRanges = mAudioTracks.mSanitizedBufferedRanges;
758
759
#if DEBUG
760
if (HasVideo()) {
761
MSE_DEBUG("after video ranges=%s",
762
DumpTimeRanges(mVideoTracks.mBufferedRanges).get());
763
}
764
if (HasAudio()) {
765
MSE_DEBUG("after audio ranges=%s",
766
DumpTimeRanges(mAudioTracks.mBufferedRanges).get());
767
}
768
#endif
769
}
770
771
void TrackBuffersManager::SegmentParserLoop() {
772
MOZ_ASSERT(OnTaskQueue());
773
774
while (true) {
775
// 1. If the input buffer is empty, then jump to the need more data step
776
// below.
777
if (!mInputBuffer || mInputBuffer->IsEmpty()) {
778
NeedMoreData();
779
return;
780
}
781
// 2. If the input buffer contains bytes that violate the SourceBuffer
782
// byte stream format specification, then run the append error algorithm
783
// with the decode error parameter set to true and abort this algorithm.
784
// TODO
785
786
// 3. Remove any bytes that the byte stream format specifications say must
787
// be ignored from the start of the input buffer. We do not remove bytes
788
// from our input buffer. Instead we enforce that our ContainerParser is
789
// able to skip over all data that is supposed to be ignored.
790
791
// 4. If the append state equals WAITING_FOR_SEGMENT, then run the following
792
// steps:
793
if (mSourceBufferAttributes->GetAppendState() ==
794
AppendState::WAITING_FOR_SEGMENT) {
795
MediaResult haveInitSegment =
796
mParser->IsInitSegmentPresent(*mInputBuffer);
797
if (NS_SUCCEEDED(haveInitSegment)) {
798
SetAppendState(AppendState::PARSING_INIT_SEGMENT);
799
if (mFirstInitializationSegmentReceived && !mChangeTypeReceived) {
800
// This is a new initialization segment. Obsolete the old one.
801
RecreateParser(false);
802
}
803
continue;
804
}
805
MediaResult haveMediaSegment =
806
mParser->IsMediaSegmentPresent(*mInputBuffer);
807
if (NS_SUCCEEDED(haveMediaSegment)) {
808
SetAppendState(AppendState::PARSING_MEDIA_SEGMENT);
809
mNewMediaSegmentStarted = true;
810
continue;
811
}
812
// We have neither an init segment nor a media segment.
813
// Check if it was invalid data.
814
if (haveInitSegment != NS_ERROR_NOT_AVAILABLE) {
815
MSE_DEBUG("Found invalid data.");
816
RejectAppend(haveInitSegment, __func__);
817
return;
818
}
819
if (haveMediaSegment != NS_ERROR_NOT_AVAILABLE) {
820
MSE_DEBUG("Found invalid data.");
821
RejectAppend(haveMediaSegment, __func__);
822
return;
823
}
824
MSE_DEBUG("Found incomplete data.");
825
NeedMoreData();
826
return;
827
}
828
829
int64_t start, end;
830
MediaResult newData =
831
mParser->ParseStartAndEndTimestamps(*mInputBuffer, start, end);
832
if (!NS_SUCCEEDED(newData) && newData.Code() != NS_ERROR_NOT_AVAILABLE) {
833
RejectAppend(newData, __func__);
834
return;
835
}
836
mProcessedInput += mInputBuffer->Length();
837
838
// 5. If the append state equals PARSING_INIT_SEGMENT, then run the
839
// following steps:
840
if (mSourceBufferAttributes->GetAppendState() ==
841
AppendState::PARSING_INIT_SEGMENT) {
842
if (mParser->InitSegmentRange().IsEmpty()) {
843
mInputBuffer.reset();
844
NeedMoreData();
845
return;
846
}
847
InitializationSegmentReceived();
848
return;
849
}
850
if (mSourceBufferAttributes->GetAppendState() ==
851
AppendState::PARSING_MEDIA_SEGMENT) {
852
// 1. If the first initialization segment received flag is false, then run
853
// the append error algorithm with the decode error parameter set to
854
// true and abort this algorithm.
855
// Or we are in the process of changeType, in which case we must first
856
// get an init segment before getting a media segment.
857
if (!mFirstInitializationSegmentReceived || mChangeTypeReceived) {
858
RejectAppend(NS_ERROR_FAILURE, __func__);
859
return;
860
}
861
862
// We can't feed some demuxers (WebMDemuxer) with data that do not have
863
// monotonizally increasing timestamps. So we check if we have a
864
// discontinuity from the previous segment parsed.
865
// If so, recreate a new demuxer to ensure that the demuxer is only fed
866
// monotonically increasing data.
867
if (mNewMediaSegmentStarted) {
868
if (NS_SUCCEEDED(newData) && mLastParsedEndTime.isSome() &&
869
start < mLastParsedEndTime.ref().ToMicroseconds()) {
870
MSE_DEBUG("Re-creating demuxer");
871
ResetDemuxingState();
872
return;
873
}
874
if (NS_SUCCEEDED(newData) || !mParser->MediaSegmentRange().IsEmpty()) {
875
if (mPendingInputBuffer) {
876
// We now have a complete media segment header. We can resume
877
// parsing the data.
878
AppendDataToCurrentInputBuffer(*mPendingInputBuffer);
879
mPendingInputBuffer.reset();
880
}
881
mNewMediaSegmentStarted = false;
882
} else {
883
// We don't have any data to demux yet, stash aside the data.
884
// This also handles the case:
885
// 2. If the input buffer does not contain a complete media segment
886
// header yet, then jump to the need more data step below.
887
if (!mPendingInputBuffer) {
888
mPendingInputBuffer = Some(MediaSpan(*mInputBuffer));
889
} else {
890
// Note we reset mInputBuffer below, so this won't end up appending
891
// the contents of mInputBuffer to itself.
892
mPendingInputBuffer->Append(*mInputBuffer);
893
}
894
895
mInputBuffer.reset();
896
NeedMoreData();
897
return;
898
}
899
}
900
901
// 3. If the input buffer contains one or more complete coded frames, then
902
// run the coded frame processing algorithm.
903
RefPtr<TrackBuffersManager> self = this;
904
CodedFrameProcessing()
905
->Then(
906
TaskQueueFromTaskQueue(), __func__,
907
[self](bool aNeedMoreData) {
908
self->mProcessingRequest.Complete();
909
if (aNeedMoreData) {
910
self->NeedMoreData();
911
} else {
912
self->ScheduleSegmentParserLoop();
913
}
914
},
915
[self](const MediaResult& aRejectValue) {
916
self->mProcessingRequest.Complete();
917
self->RejectAppend(aRejectValue, __func__);
918
})
919
->Track(mProcessingRequest);
920
return;
921
}
922
}
923
}
924
925
void TrackBuffersManager::NeedMoreData() {
926
MSE_DEBUG("");
927
MOZ_DIAGNOSTIC_ASSERT(mCurrentTask &&
928
mCurrentTask->GetType() ==
929
SourceBufferTask::Type::AppendBuffer);
930
MOZ_DIAGNOSTIC_ASSERT(mSourceBufferAttributes);
931
932
mCurrentTask->As<AppendBufferTask>()->mPromise.Resolve(
933
SourceBufferTask::AppendBufferResult(mActiveTrack,
934
*mSourceBufferAttributes),
935
__func__);
936
mSourceBufferAttributes = nullptr;
937
mCurrentTask = nullptr;
938
ProcessTasks();
939
}
940
941
void TrackBuffersManager::RejectAppend(const MediaResult& aRejectValue,
942
const char* aName) {
943
MSE_DEBUG("rv=%" PRIu32, static_cast<uint32_t>(aRejectValue.Code()));
944
MOZ_DIAGNOSTIC_ASSERT(mCurrentTask &&
945
mCurrentTask->GetType() ==
946
SourceBufferTask::Type::AppendBuffer);
947
948
mCurrentTask->As<AppendBufferTask>()->mPromise.Reject(aRejectValue, __func__);
949
mSourceBufferAttributes = nullptr;
950
mCurrentTask = nullptr;
951
ProcessTasks();
952
}
953
954
void TrackBuffersManager::ScheduleSegmentParserLoop() {
955
MOZ_ASSERT(OnTaskQueue());
956
TaskQueueFromTaskQueue()->Dispatch(
957
NewRunnableMethod("TrackBuffersManager::SegmentParserLoop", this,
958
&TrackBuffersManager::SegmentParserLoop));
959
}
960
961
void TrackBuffersManager::ShutdownDemuxers() {
962
if (mVideoTracks.mDemuxer) {
963
mVideoTracks.mDemuxer->BreakCycles();
964
mVideoTracks.mDemuxer = nullptr;
965
}
966
if (mAudioTracks.mDemuxer) {
967
mAudioTracks.mDemuxer->BreakCycles();
968
mAudioTracks.mDemuxer = nullptr;
969
}
970
// We shouldn't change mInputDemuxer while a demuxer init/reset request is
971
// being processed. See bug 1239983.
972
MOZ_DIAGNOSTIC_ASSERT(!mDemuxerInitRequest.Exists());
973
mInputDemuxer = nullptr;
974
mLastParsedEndTime.reset();
975
}
976
977
void TrackBuffersManager::CreateDemuxerforMIMEType() {
978
ShutdownDemuxers();
979
980
if (mType.Type() == MEDIAMIMETYPE(VIDEO_WEBM) ||
981
mType.Type() == MEDIAMIMETYPE(AUDIO_WEBM)) {
982
mInputDemuxer =
983
new WebMDemuxer(mCurrentInputBuffer, true /* IsMediaSource*/);
984
DDLINKCHILD("demuxer", mInputDemuxer.get());
985
return;
986
}
987
988
#ifdef MOZ_FMP4
989
if (mType.Type() == MEDIAMIMETYPE(VIDEO_MP4) ||
990
mType.Type() == MEDIAMIMETYPE(AUDIO_MP4)) {
991
mInputDemuxer = new MP4Demuxer(mCurrentInputBuffer);
992
DDLINKCHILD("demuxer", mInputDemuxer.get());
993
return;
994
}
995
#endif
996
NS_WARNING("Not supported (yet)");
997
}
998
999
// We reset the demuxer by creating a new one and initializing it.
1000
void TrackBuffersManager::ResetDemuxingState() {
1001
MOZ_ASSERT(OnTaskQueue());
1002
MOZ_ASSERT(mParser && mParser->HasInitData());
1003
RecreateParser(true);
1004
mCurrentInputBuffer = new SourceBufferResource();
1005
// The demuxer isn't initialized yet ; we don't want to notify it
1006
// that data has been appended yet ; so we simply append the init segment
1007
// to the resource.
1008
mCurrentInputBuffer->AppendData(mParser->InitData());
1009
CreateDemuxerforMIMEType();
1010
if (!mInputDemuxer) {
1011
RejectAppend(NS_ERROR_FAILURE, __func__);
1012
return;
1013
}
1014
mInputDemuxer->Init()
1015
->Then(TaskQueueFromTaskQueue(), __func__, this,
1016
&TrackBuffersManager::OnDemuxerResetDone,
1017
&TrackBuffersManager::OnDemuxerInitFailed)
1018
->Track(mDemuxerInitRequest);
1019
}
1020
1021
void TrackBuffersManager::OnDemuxerResetDone(const MediaResult& aResult) {
1022
MOZ_ASSERT(OnTaskQueue());
1023
mDemuxerInitRequest.Complete();
1024
1025
if (NS_FAILED(aResult) && StaticPrefs::media_playback_warnings_as_errors()) {
1026
RejectAppend(aResult, __func__);
1027
return;
1028
}
1029
1030
// mInputDemuxer shouldn't have been destroyed while a demuxer init/reset
1031
// request was being processed. See bug 1239983.
1032
MOZ_DIAGNOSTIC_ASSERT(mInputDemuxer);
1033
1034
if (aResult != NS_OK && mParentDecoder) {
1035
RefPtr<TrackBuffersManager> self = this;
1036
mAbstractMainThread->Dispatch(NS_NewRunnableFunction(
1037
"TrackBuffersManager::OnDemuxerResetDone", [self, aResult]() {
1038
if (self->mParentDecoder && self->mParentDecoder->GetOwner()) {
1039
self->mParentDecoder->GetOwner()->DecodeWarning(aResult);
1040
}
1041
}));
1042
}
1043
1044
// Recreate track demuxers.
1045
uint32_t numVideos = mInputDemuxer->GetNumberTracks(TrackInfo::kVideoTrack);
1046
if (numVideos) {
1047
// We currently only handle the first video track.
1048
mVideoTracks.mDemuxer =
1049
mInputDemuxer->GetTrackDemuxer(TrackInfo::kVideoTrack, 0);
1050
MOZ_ASSERT(mVideoTracks.mDemuxer);
1051
DDLINKCHILD("video demuxer", mVideoTracks.mDemuxer.get());
1052
}
1053
1054
uint32_t numAudios = mInputDemuxer->GetNumberTracks(TrackInfo::kAudioTrack);
1055
if (numAudios) {
1056
// We currently only handle the first audio track.
1057
mAudioTracks.mDemuxer =
1058
mInputDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0);
1059
MOZ_ASSERT(mAudioTracks.mDemuxer);
1060
DDLINKCHILD("audio demuxer", mAudioTracks.mDemuxer.get());
1061
}
1062
1063
if (mPendingInputBuffer) {
1064
// We had a partial media segment header stashed aside.
1065
// Reparse its content so we can continue parsing the current input buffer.
1066
int64_t start, end;
1067
mParser->ParseStartAndEndTimestamps(*mPendingInputBuffer, start, end);
1068
mProcessedInput += mPendingInputBuffer->Length();
1069
}
1070
1071
SegmentParserLoop();
1072
}
1073
1074
void TrackBuffersManager::AppendDataToCurrentInputBuffer(
1075
const MediaSpan& aData) {
1076
MOZ_ASSERT(mCurrentInputBuffer);
1077
mCurrentInputBuffer->AppendData(aData);
1078
mInputDemuxer->NotifyDataArrived();
1079
}
1080
1081
void TrackBuffersManager::InitializationSegmentReceived() {
1082
MOZ_ASSERT(OnTaskQueue());
1083
MOZ_ASSERT(mParser->HasCompleteInitData());
1084
1085
int64_t endInit = mParser->InitSegmentRange().mEnd;
1086
if (mInputBuffer->Length() > mProcessedInput ||
1087
int64_t(mProcessedInput - mInputBuffer->Length()) > endInit) {
1088
// Something is not quite right with the data appended. Refuse it.
1089
RejectAppend(MediaResult(NS_ERROR_FAILURE,
1090
"Invalid state following initialization segment"),
1091
__func__);
1092
return;
1093
}
1094
1095
mCurrentInputBuffer = new SourceBufferResource();
1096
// The demuxer isn't initialized yet ; we don't want to notify it
1097
// that data has been appended yet ; so we simply append the init segment
1098
// to the resource.
1099
mCurrentInputBuffer->AppendData(mParser->InitData());
1100
uint32_t length = endInit - (mProcessedInput - mInputBuffer->Length());
1101
MOZ_RELEASE_ASSERT(length <= mInputBuffer->Length());
1102
mInputBuffer->RemoveFront(length);
1103
CreateDemuxerforMIMEType();
1104
if (!mInputDemuxer) {
1105
NS_WARNING("TODO type not supported");
1106
RejectAppend(NS_ERROR_DOM_NOT_SUPPORTED_ERR, __func__);
1107
return;
1108
}
1109
mInputDemuxer->Init()
1110
->Then(TaskQueueFromTaskQueue(), __func__, this,
1111
&TrackBuffersManager::OnDemuxerInitDone,
1112
&TrackBuffersManager::OnDemuxerInitFailed)
1113
->Track(mDemuxerInitRequest);
1114
}
1115
1116
void TrackBuffersManager::OnDemuxerInitDone(const MediaResult& aResult) {
1117
MOZ_ASSERT(OnTaskQueue());
1118
MOZ_DIAGNOSTIC_ASSERT(mInputDemuxer, "mInputDemuxer has been destroyed");
1119
1120
mDemuxerInitRequest.Complete();
1121
1122
if (NS_FAILED(aResult) && StaticPrefs::media_playback_warnings_as_errors()) {
1123
RejectAppend(aResult, __func__);
1124
return;
1125
}
1126
1127
MediaInfo info;
1128
1129
uint32_t numVideos = mInputDemuxer->GetNumberTracks(TrackInfo::kVideoTrack);
1130
if (numVideos) {
1131
// We currently only handle the first video track.
1132
mVideoTracks.mDemuxer =
1133
mInputDemuxer->GetTrackDemuxer(TrackInfo::kVideoTrack, 0);
1134
MOZ_ASSERT(mVideoTracks.mDemuxer);
1135
DDLINKCHILD("video demuxer", mVideoTracks.mDemuxer.get());
1136
info.mVideo = *mVideoTracks.mDemuxer->GetInfo()->GetAsVideoInfo();
1137
info.mVideo.mTrackId = 2;
1138
}
1139
1140
uint32_t numAudios = mInputDemuxer->GetNumberTracks(TrackInfo::kAudioTrack);
1141
if (numAudios) {
1142
// We currently only handle the first audio track.
1143
mAudioTracks.mDemuxer =
1144
mInputDemuxer->GetTrackDemuxer(TrackInfo::kAudioTrack, 0);
1145
MOZ_ASSERT(mAudioTracks.mDemuxer);
1146
DDLINKCHILD("audio demuxer", mAudioTracks.mDemuxer.get());
1147
info.mAudio = *mAudioTracks.mDemuxer->GetInfo()->GetAsAudioInfo();
1148
info.mAudio.mTrackId = 1;
1149
}
1150
1151
int64_t videoDuration =
1152
numVideos ? info.mVideo.mDuration.ToMicroseconds() : 0;
1153
int64_t audioDuration =
1154
numAudios ? info.mAudio.mDuration.ToMicroseconds() : 0;
1155
1156
int64_t duration = std::max(videoDuration, audioDuration);
1157
// 1. Update the duration attribute if it currently equals NaN.
1158
// Those steps are performed by the MediaSourceDecoder::SetInitialDuration
1159
mAbstractMainThread->Dispatch(NewRunnableMethod<int64_t>(
1160
"MediaSourceDecoder::SetInitialDuration", mParentDecoder.get(),
1161
&MediaSourceDecoder::SetInitialDuration, duration ? duration : -1));
1162
1163
// 2. If the initialization segment has no audio, video, or text tracks, then
1164
// run the append error algorithm with the decode error parameter set to true
1165
// and abort these steps.
1166
if (!numVideos && !numAudios) {
1167
RejectAppend(NS_ERROR_FAILURE, __func__);
1168
return;
1169
}
1170
1171
// 3. If the first initialization segment received flag is true, then run the
1172
// following steps:
1173
if (mFirstInitializationSegmentReceived) {
1174
if (numVideos != mVideoTracks.mNumTracks ||
1175
numAudios != mAudioTracks.mNumTracks) {
1176
RejectAppend(NS_ERROR_FAILURE, __func__);
1177
return;
1178
}
1179
// 1. If more than one track for a single type are present (ie 2 audio
1180
// tracks), then the Track IDs match the ones in the first initialization
1181
// segment.
1182
// TODO
1183
// 2. Add the appropriate track descriptions from this initialization
1184
// segment to each of the track buffers.
1185
// TODO
1186
// 3. Set the need random access point flag on all track buffers to true.
1187
mVideoTracks.mNeedRandomAccessPoint = true;
1188
mAudioTracks.mNeedRandomAccessPoint = true;
1189
}
1190
1191
// Check if we've received the same init data again. Some streams will
1192
// resend the same data. In these cases we don't need to change the stream
1193
// id as it's the same stream. Doing so would recreate decoders, possibly
1194
// leading to gaps in audio and/or video (see bug 1450952).
1195
//
1196
// It's possible to have the different binary representations for the same
1197
// logical init data. If this case is encountered in the wild then these
1198
// checks could be revised to compare MediaInfo rather than init segment
1199
// bytes.
1200
// For audio only source buffer we can check that only the AudioInfo has
1201
// changed.
1202
bool isRepeatInitData =
1203
!mChangeTypeReceived && mInitData &&
1204
((*mInitData.get() == *mParser->InitData()) ||
1205
(numVideos == 0 && numAudios > 0 && mAudioTracks.mLastInfo &&
1206
*mAudioTracks.mLastInfo->GetAsAudioInfo() == info.mAudio));
1207
1208
MOZ_ASSERT(mFirstInitializationSegmentReceived || !isRepeatInitData,
1209
"Should never detect repeat init data for first segment!");
1210
1211
// If we have new init data we configure and set track info as needed. If we
1212
// have repeat init data we carry forward our existing track info.
1213
if (!isRepeatInitData) {
1214
// Increase our stream id.
1215
uint32_t streamID = sStreamSourceID++;
1216
1217
// 4. Let active track flag equal false.
1218
bool activeTrack = false;
1219
1220
// 5. If the first initialization segment received flag is false, then run
1221
// the following steps:
1222
if (!mFirstInitializationSegmentReceived) {
1223
mAudioTracks.mNumTracks = numAudios;
1224
// TODO:
1225
// 1. If the initialization segment contains tracks with codecs the user
1226
// agent does not support, then run the append error algorithm with the
1227
// decode error parameter set to true and abort these steps.
1228
1229
// 2. For each audio track in the initialization segment, run following
1230
// steps: for (uint32_t i = 0; i < numAudios; i++) {
1231
if (numAudios) {
1232
// 1. Let audio byte stream track ID be the Track ID for the current
1233
// track being processed.
1234
// 2. Let audio language be a BCP 47 language tag for the language
1235
// specified in the initialization segment for this track or an empty
1236
// string if no language info is present.
1237
// 3. If audio language equals an empty string or the 'und' BCP 47
1238
// value, then run the default track language algorithm with
1239
// byteStreamTrackID set to audio byte stream track ID and type set to
1240
// "audio" and assign the value returned by the algorithm to audio
1241
// language.
1242
// 4. Let audio label be a label specified in the initialization segment
1243
// for this track or an empty string if no label info is present.
1244
// 5. If audio label equals an empty string, then run the default track
1245
// label algorithm with byteStreamTrackID set to audio byte stream track
1246
// ID and type set to "audio" and assign the value returned by the
1247
// algorithm to audio label.
1248
// 6. Let audio kinds be an array of kind strings specified in the
1249
// initialization segment for this track or an empty array if no kind
1250
// information is provided.
1251
// 7. If audio kinds equals an empty array, then run the default track
1252
// kinds algorithm with byteStreamTrackID set to audio byte stream track
1253
// ID and type set to "audio" and assign the value returned by the
1254
// algorithm to audio kinds.
1255
// 8. For each value in audio kinds, run the following steps:
1256
// 1. Let current audio kind equal the value from audio kinds for this
1257
// iteration of the loop.
1258
// 2. Let new audio track be a new AudioTrack object.
1259
// 3. Generate a unique ID and assign it to the id property on new
1260
// audio track.
1261
// 4. Assign audio language to the language property on new audio
1262
// track.
1263
// 5. Assign audio label to the label property on new audio track.
1264
// 6. Assign current audio kind to the kind property on new audio
1265
// track.
1266
// 7. If audioTracks.length equals 0, then run the following steps:
1267
// 1. Set the enabled property on new audio track to true.
1268
// 2. Set active track flag to true.
1269
activeTrack = true;
1270
// 8. Add new audio track to the audioTracks attribute on this
1271
// SourceBuffer object.
1272
// 9. Queue a task to fire a trusted event named addtrack, that does
1273
// not bubble and is not cancelable, and that uses the TrackEvent
1274
// interface, at the AudioTrackList object referenced by the
1275
// audioTracks attribute on this SourceBuffer object.
1276
// 10. Add new audio track to the audioTracks attribute on the
1277
// HTMLMediaElement.
1278
// 11. Queue a task to fire a trusted event named addtrack, that does
1279
// not bubble and is not cancelable, and that uses the TrackEvent
1280
// interface, at the AudioTrackList object referenced by the
1281
// audioTracks attribute on the HTMLMediaElement.
1282
mAudioTracks.mBuffers.AppendElement(TrackBuffer());
1283
// 10. Add the track description for this track to the track buffer.
1284
mAudioTracks.mInfo = new TrackInfoSharedPtr(info.mAudio, streamID);
1285
mAudioTracks.mLastInfo = mAudioTracks.mInfo;
1286
}
1287
1288
mVideoTracks.mNumTracks = numVideos;
1289
// 3. For each video track in the initialization segment, run following
1290
// steps: for (uint32_t i = 0; i < numVideos; i++) {
1291
if (numVideos) {
1292
// 1. Let video byte stream track ID be the Track ID for the current
1293
// track being processed.
1294
// 2. Let video language be a BCP 47 language tag for the language
1295
// specified in the initialization segment for this track or an empty
1296
// string if no language info is present.
1297
// 3. If video language equals an empty string or the 'und' BCP 47
1298
// value, then run the default track language algorithm with
1299
// byteStreamTrackID set to video byte stream track ID and type set to
1300
// "video" and assign the value returned by the algorithm to video
1301
// language.
1302
// 4. Let video label be a label specified in the initialization segment
1303
// for this track or an empty string if no label info is present.
1304
// 5. If video label equals an empty string, then run the default track
1305
// label algorithm with byteStreamTrackID set to video byte stream track
1306
// ID and type set to "video" and assign the value returned by the
1307
// algorithm to video label.
1308
// 6. Let video kinds be an array of kind strings specified in the
1309
// initialization segment for this track or an empty array if no kind
1310
// information is provided.
1311
// 7. If video kinds equals an empty array, then run the default track
1312
// kinds algorithm with byteStreamTrackID set to video byte stream track
1313
// ID and type set to "video" and assign the value returned by the
1314
// algorithm to video kinds.
1315
// 8. For each value in video kinds, run the following steps:
1316
// 1. Let current video kind equal the value from video kinds for this
1317
// iteration of the loop.
1318
// 2. Let new video track be a new VideoTrack object.
1319
// 3. Generate a unique ID and assign it to the id property on new
1320
// video track.
1321
// 4. Assign video language to the language property on new video
1322
// track.
1323
// 5. Assign video label to the label property on new video track.
1324
// 6. Assign current video kind to the kind property on new video
1325
// track.
1326
// 7. If videoTracks.length equals 0, then run the following steps:
1327
// 1. Set the selected property on new video track to true.
1328
// 2. Set active track flag to true.
1329
activeTrack = true;
1330
// 8. Add new video track to the videoTracks attribute on this
1331
// SourceBuffer object.
1332
// 9. Queue a task to fire a trusted event named addtrack, that does
1333
// not bubble and is not cancelable, and that uses the TrackEvent
1334
// interface, at the VideoTrackList object referenced by the
1335
// videoTracks attribute on this SourceBuffer object.
1336
// 10. Add new video track to the videoTracks attribute on the
1337
// HTMLMediaElement.
1338
// 11. Queue a task to fire a trusted event named addtrack, that does
1339
// not bubble and is not cancelable, and that uses the TrackEvent
1340
// interface, at the VideoTrackList object referenced by the
1341
// videoTracks attribute on the HTMLMediaElement.
1342
mVideoTracks.mBuffers.AppendElement(TrackBuffer());
1343
// 10. Add the track description for this track to the track buffer.
1344
mVideoTracks.mInfo = new TrackInfoSharedPtr(info.mVideo, streamID);
1345
mVideoTracks.mLastInfo = mVideoTracks.mInfo;
1346
}
1347
// 4. For each text track in the initialization segment, run following
1348
// steps:
1349
// 5. If active track flag equals true, then run the following steps:
1350
// This is handled by SourceBuffer once the promise is resolved.
1351
if (activeTrack) {
1352
mActiveTrack = true;
1353
}
1354
1355
// 6. Set first initialization segment received flag to true.
1356
mFirstInitializationSegmentReceived = true;
1357
} else {
1358
mAudioTracks.mLastInfo = new TrackInfoSharedPtr(info.mAudio, streamID);
1359
mVideoTracks.mLastInfo = new TrackInfoSharedPtr(info.mVideo, streamID);
1360
}
1361
1362
UniquePtr<EncryptionInfo> crypto = mInputDemuxer->GetCrypto();
1363
if (crypto && crypto->IsEncrypted()) {
1364
// Try and dispatch 'encrypted'. Won't go if ready state still
1365
// HAVE_NOTHING.
1366
for (uint32_t i = 0; i < crypto->mInitDatas.Length(); i++) {
1367
nsCOMPtr<nsIRunnable> r = new DispatchKeyNeededEvent(
1368
mParentDecoder, crypto->mInitDatas[i].mInitData,
1369
crypto->mInitDatas[i].mType);
1370
mAbstractMainThread->Dispatch(r.forget());
1371
}
1372
info.mCrypto = *crypto;
1373
// We clear our crypto init data array, so the MediaFormatReader will
1374
// not emit an encrypted event for the same init data again.
1375
info.mCrypto.mInitDatas.Clear();
1376
}
1377
1378
{
1379
MutexAutoLock mut(mMutex);
1380
mInfo = info;
1381
}
1382
1383
// We now have a valid init data ; we can store it for later use.
1384
mInitData = mParser->InitData();
1385
}
1386
1387
// We have now completed the changeType operation.
1388
mChangeTypeReceived = false;
1389
1390
// 3. Remove the initialization segment bytes from the beginning of the input
1391
// buffer. This step has already been done in InitializationSegmentReceived
1392
// when we transferred the content into mCurrentInputBuffer.
1393
mCurrentInputBuffer->EvictAll();
1394
mInputDemuxer->NotifyDataRemoved();
1395
RecreateParser(true);
1396
1397
// 4. Set append state to WAITING_FOR_SEGMENT.
1398
SetAppendState(AppendState::WAITING_FOR_SEGMENT);
1399
// 5. Jump to the loop top step above.
1400
ScheduleSegmentParserLoop();
1401
1402
if (aResult != NS_OK && mParentDecoder) {
1403
RefPtr<TrackBuffersManager> self = this;
1404
mAbstractMainThread->Dispatch(NS_NewRunnableFunction(
1405
"TrackBuffersManager::OnDemuxerInitDone", [self, aResult]() {
1406
if (self->mParentDecoder && self->mParentDecoder->GetOwner()) {
1407
self->mParentDecoder->GetOwner()->DecodeWarning(aResult);
1408
}
1409
}));
1410
}
1411
}
1412
1413
void TrackBuffersManager::OnDemuxerInitFailed(const MediaResult& aError) {
1414
MOZ_ASSERT(aError != NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA);
1415
mDemuxerInitRequest.Complete();
1416
1417
RejectAppend(aError, __func__);
1418
}
1419
1420
RefPtr<TrackBuffersManager::CodedFrameProcessingPromise>
1421
TrackBuffersManager::CodedFrameProcessing() {
1422
MOZ_ASSERT(OnTaskQueue());
1423
MOZ_ASSERT(mProcessingPromise.IsEmpty());
1424
1425
MediaByteRange mediaRange = mParser->MediaSegmentRange();
1426
if (mediaRange.IsEmpty()) {
1427
AppendDataToCurrentInputBuffer(*mInputBuffer);
1428
mInputBuffer.reset();
1429
} else {
1430
MOZ_ASSERT(mProcessedInput >= mInputBuffer->Length());
1431
if (int64_t(mProcessedInput - mInputBuffer->Length()) > mediaRange.mEnd) {
1432
// Something is not quite right with the data appended. Refuse it.
1433
// This would typically happen if the previous media segment was partial
1434
// yet a new complete media segment was added.
1435
return CodedFrameProcessingPromise::CreateAndReject(NS_ERROR_FAILURE,
1436
__func__);
1437
}
1438
// The mediaRange is offset by the init segment position previously added.
1439
uint32_t length =
1440
mediaRange.mEnd - (mProcessedInput - mInputBuffer->Length());
1441
if (!length) {
1442
// We've completed our earlier media segment and no new data is to be
1443
// processed. This happens with some containers that can't detect that a
1444
// media segment is ending until a new one starts.
1445
RefPtr<CodedFrameProcessingPromise> p =
1446
mProcessingPromise.Ensure(__func__);
1447
CompleteCodedFrameProcessing();
1448
return p;
1449
}
1450
AppendDataToCurrentInputBuffer(mInputBuffer->To(length));
1451
mInputBuffer->RemoveFront(length);
1452
}
1453
1454
RefPtr<CodedFrameProcessingPromise> p = mProcessingPromise.Ensure(__func__);
1455
1456
DoDemuxVideo();
1457
1458
return p;
1459
}
1460
1461
void TrackBuffersManager::OnDemuxFailed(TrackType aTrack,
1462
const MediaResult& aError) {
1463
MOZ_ASSERT(OnTaskQueue());
1464
MSE_DEBUG("Failed to demux %s, failure:%s",
1465
aTrack == TrackType::kVideoTrack ? "video" : "audio",
1466
aError.ErrorName().get());
1467
switch (aError.Code()) {
1468
case NS_ERROR_DOM_MEDIA_END_OF_STREAM:
1469
case NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA:
1470
if (aTrack == TrackType::kVideoTrack) {
1471
DoDemuxAudio();
1472
} else {
1473
CompleteCodedFrameProcessing();
1474
}
1475
break;
1476
default:
1477
RejectProcessing(aError, __func__);
1478
break;
1479
}
1480
}
1481
1482
void TrackBuffersManager::DoDemuxVideo() {
1483
MOZ_ASSERT(OnTaskQueue());
1484
if (!HasVideo()) {
1485
DoDemuxAudio();
1486
return;
1487
}
1488
mVideoTracks.mDemuxer->GetSamples(-1)
1489
->Then(TaskQueueFromTaskQueue(), __func__, this,
1490
&TrackBuffersManager::OnVideoDemuxCompleted,
1491
&TrackBuffersManager::OnVideoDemuxFailed)
1492
->Track(mVideoTracks.mDemuxRequest);
1493
}
1494
1495
void TrackBuffersManager::MaybeDispatchEncryptedEvent(
1496
const nsTArray<RefPtr<MediaRawData>>& aSamples) {
1497
// Try and dispatch 'encrypted'. Won't go if ready state still HAVE_NOTHING.
1498
for (const RefPtr<MediaRawData>& sample : aSamples) {
1499
for (const nsTArray<uint8_t>& initData : sample->mCrypto.mInitDatas) {
1500
nsCOMPtr<nsIRunnable> r = new DispatchKeyNeededEvent(
1501
mParentDecoder, initData, sample->mCrypto.mInitDataType);
1502
mAbstractMainThread->Dispatch(r.forget());
1503
}
1504
}
1505
}
1506
1507
void TrackBuffersManager::OnVideoDemuxCompleted(
1508
RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
1509
MOZ_ASSERT(OnTaskQueue());
1510
MSE_DEBUG("%zu video samples demuxed", aSamples->GetSamples().Length());
1511
mVideoTracks.mDemuxRequest.Complete();
1512
mVideoTracks.mQueuedSamples.AppendElements(aSamples->GetSamples());
1513
1514
MaybeDispatchEncryptedEvent(aSamples->GetSamples());
1515
DoDemuxAudio();
1516
}
1517
1518
void TrackBuffersManager::DoDemuxAudio() {
1519
MOZ_ASSERT(OnTaskQueue());
1520
if (!HasAudio()) {
1521
CompleteCodedFrameProcessing();
1522
return;
1523
}
1524
mAudioTracks.mDemuxer->GetSamples(-1)
1525
->Then(TaskQueueFromTaskQueue(), __func__, this,
1526
&TrackBuffersManager::OnAudioDemuxCompleted,
1527
&TrackBuffersManager::OnAudioDemuxFailed)
1528
->Track(mAudioTracks.mDemuxRequest);
1529
}
1530
1531
void TrackBuffersManager::OnAudioDemuxCompleted(
1532
RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples) {
1533
MOZ_ASSERT(OnTaskQueue());
1534
MSE_DEBUG("%zu audio samples demuxed", aSamples->GetSamples().Length());
1535
mAudioTracks.mDemuxRequest.Complete();
1536
mAudioTracks.mQueuedSamples.AppendElements(aSamples->GetSamples());
1537
CompleteCodedFrameProcessing();
1538
1539
MaybeDispatchEncryptedEvent(aSamples->GetSamples());
1540
}
1541
1542
void TrackBuffersManager::CompleteCodedFrameProcessing() {
1543
MOZ_ASSERT(OnTaskQueue());
1544
1545
// 1. For each coded frame in the media segment run the following steps:
1546
// Coded Frame Processing steps 1.1 to 1.21.
1547
1548
if (mSourceBufferAttributes->GetAppendMode() ==
1549
SourceBufferAppendMode::Sequence &&
1550
mVideoTracks.mQueuedSamples.Length() &&
1551
mAudioTracks.mQueuedSamples.Length()) {
1552
// When we are in sequence mode, the order in which we process the frames is
1553
// important as it determines the future value of timestampOffset.
1554
// So we process the earliest sample first. See bug 1293576.
1555
TimeInterval videoInterval =
1556
PresentationInterval(mVideoTracks.mQueuedSamples);
1557
TimeInterval audioInterval =
1558
PresentationInterval(mAudioTracks.mQueuedSamples);
1559
if (audioInterval.mStart < videoInterval.mStart) {
1560
ProcessFrames(mAudioTracks.mQueuedSamples, mAudioTracks);
1561
ProcessFrames(mVideoTracks.mQueuedSamples, mVideoTracks);
1562
} else {
1563
ProcessFrames(mVideoTracks.mQueuedSamples, mVideoTracks);
1564
ProcessFrames(mAudioTracks.mQueuedSamples, mAudioTracks);
1565
}
1566
} else {
1567
ProcessFrames(mVideoTracks.mQueuedSamples, mVideoTracks);
1568
ProcessFrames(mAudioTracks.mQueuedSamples, mAudioTracks);
1569
}
1570
1571
#if defined(DEBUG)
1572
if (HasVideo()) {
1573
const auto& track = mVideoTracks.GetTrackBuffer();
1574
MOZ_ASSERT(track.IsEmpty() || track[0]->mKeyframe);
1575
for (uint32_t i = 1; i < track.Length(); i++) {
1576
MOZ_ASSERT(
1577
(track[i - 1]->mTrackInfo->GetID() == track[i]->mTrackInfo->GetID() &&
1578
track[i - 1]->mTimecode <= track[i]->mTimecode) ||
1579
track[i]->mKeyframe);
1580
}
1581
}
1582
if (HasAudio()) {
1583
const auto& track = mAudioTracks.GetTrackBuffer();
1584
MOZ_ASSERT(track.IsEmpty() || track[0]->mKeyframe);
1585
for (uint32_t i = 1; i < track.Length(); i++) {
1586
MOZ_ASSERT(
1587
(track[i - 1]->mTrackInfo->GetID() == track[i]->mTrackInfo->GetID() &&
1588
track[i - 1]->mTimecode <= track[i]->mTimecode) ||
1589
track[i]->mKeyframe);
1590
}
1591
}
1592
#endif
1593
1594
mVideoTracks.mQueuedSamples.Clear();
1595
mAudioTracks.mQueuedSamples.Clear();
1596
1597
UpdateBufferedRanges();
1598
1599
// Update our reported total size.
1600
mSizeSourceBuffer = mVideoTracks.mSizeBuffer + mAudioTracks.mSizeBuffer;
1601
1602
// Return to step 6.4 of Segment Parser Loop algorithm
1603
// 4. If this SourceBuffer is full and cannot accept more media data, then set
1604
// the buffer full flag to true.
1605
if (mSizeSourceBuffer >= EvictionThreshold()) {
1606
mBufferFull = true;
1607
}
1608
1609
// 5. If the input buffer does not contain a complete media segment, then jump
1610
// to the need more data step below.
1611
if (mParser->MediaSegmentRange().IsEmpty()) {
1612
ResolveProcessing(true, __func__);
1613
return;
1614
}
1615
1616
mLastParsedEndTime = Some(std::max(mAudioTracks.mLastParsedEndTime,
1617
mVideoTracks.mLastParsedEndTime));
1618
1619
// 6. Remove the media segment bytes from the beginning of the input buffer.
1620
// Clear our demuxer from any already processed data.
1621
int64_t safeToEvict =
1622
std::min(HasVideo() ? mVideoTracks.mDemuxer->GetEvictionOffset(
1623
mVideoTracks.mLastParsedEndTime)
1624
: INT64_MAX,
1625
HasAudio() ? mAudioTracks.mDemuxer->GetEvictionOffset(
1626
mAudioTracks.mLastParsedEndTime)
1627
: INT64_MAX);
1628
mCurrentInputBuffer->EvictBefore(safeToEvict);
1629
1630
mInputDemuxer->NotifyDataRemoved();
1631
RecreateParser(true);
1632
1633
// 7. Set append state to WAITING_FOR_SEGMENT.
1634
SetAppendState(AppendState::WAITING_FOR_SEGMENT);
1635
1636
// 8. Jump to the loop top step above.
1637
ResolveProcessing(false, __func__);
1638
}
1639
1640
void TrackBuffersManager::RejectProcessing(const MediaResult& aRejectValue,
1641
const char* aName) {
1642
mProcessingPromise.RejectIfExists(aRejectValue, __func__);
1643
}
1644
1645
void TrackBuffersManager::ResolveProcessing(bool aResolveValue,
1646
const char* aName) {
1647
mProcessingPromise.ResolveIfExists(aResolveValue, __func__);
1648
}
1649
1650
void TrackBuffersManager::CheckSequenceDiscontinuity(
1651
const TimeUnit& aPresentationTime) {
1652
if (mSourceBufferAttributes->GetAppendMode() ==
1653
SourceBufferAppendMode::Sequence &&
1654
mSourceBufferAttributes->HaveGroupStartTimestamp()) {
1655
mSourceBufferAttributes->SetTimestampOffset(
1656
mSourceBufferAttributes->GetGroupStartTimestamp() - aPresentationTime);
1657
mSourceBufferAttributes->SetGroupEndTimestamp(
1658
mSourceBufferAttributes->GetGroupStartTimestamp());
1659
mVideoTracks.mNeedRandomAccessPoint = true;
1660
mAudioTracks.mNeedRandomAccessPoint = true;
1661
mSourceBufferAttributes->ResetGroupStartTimestamp();
1662
}
1663
}
1664
1665
TimeInterval TrackBuffersManager::PresentationInterval(
1666
const TrackBuffer& aSamples) const {
1667
TimeInterval presentationInterval =
1668
TimeInterval(aSamples[0]->mTime, aSamples[0]->GetEndTime());
1669
1670
for (uint32_t i = 1; i < aSamples.Length(); i++) {
1671
auto& sample = aSamples[i];
1672
presentationInterval = presentationInterval.Span(
1673
TimeInterval(sample->mTime, sample->GetEndTime()));
1674
}
1675
return presentationInterval;
1676
}
1677
1678
void TrackBuffersManager::ProcessFrames(TrackBuffer& aSamples,
1679
TrackData& aTrackData) {
1680
if (!aSamples.Length()) {
1681
return;
1682
}
1683
1684
// 1. If generate timestamps flag equals true
1685
// Let presentation timestamp equal 0.
1686
// Otherwise
1687
// Let presentation timestamp be a double precision floating point
1688
// representation of the coded frame's presentation timestamp in seconds.
1689
TimeUnit presentationTimestamp = mSourceBufferAttributes->mGenerateTimestamps
1690
? TimeUnit::Zero()
1691
: aSamples[0]->mTime;
1692
1693
// 3. If mode equals "sequence" and group start timestamp is set, then run the
1694
// following steps:
1695
CheckSequenceDiscontinuity(presentationTimestamp);
1696
1697
// 5. Let track buffer equal the track buffer that the coded frame will be
1698
// added to.
1699
auto& trackBuffer = aTrackData;
1700
1701
TimeIntervals samplesRange;
1702
uint32_t sizeNewSamples = 0;
1703
TrackBuffer samples; // array that will contain the frames to be added
1704
// to our track buffer.
1705
1706
// We assume that no frames are contiguous within a media segment and as such
1707
// don't need to check for discontinuity except for the first frame and should
1708
// a frame be ignored due to the target window.
1709
bool needDiscontinuityCheck = true;
1710
1711
// Highest presentation time seen in samples block.
1712
TimeUnit highestSampleTime;
1713
1714
if (aSamples.Length()) {
1715
aTrackData.mLastParsedEndTime = TimeUnit();
1716
}
1717
1718
auto addToSamples = [&](MediaRawData* aSample,
1719
const TimeInterval& aInterval) {
1720
aSample->mTime = aInterval.mStart;
1721
aSample->mDuration = aInterval.Length();
1722
aSample->mTrackInfo = trackBuffer.mLastInfo;
1723
MOZ_DIAGNOSTIC_ASSERT(aSample->HasValidTime());
1724
samplesRange += aInterval;
1725
sizeNewSamples += aSample->ComputedSizeOfIncludingThis();
1726
samples.AppendElement(aSample);
1727
};
1728
1729
// Will be set to the last frame dropped due to being outside mAppendWindow.
1730
// It will be added prior the first following frame which can be added to the
1731
// track buffer.
1732
// This sample will be set with a duration of only 1us which will cause it to
1733
// be dropped once returned by the decoder.
1734
// This sample is required to "prime" the decoder so that the following frame
1735
// can be fully decoded.
1736
RefPtr<MediaRawData> previouslyDroppedSample;
1737
for (auto& sample : aSamples) {
1738
const TimeUnit sampleEndTime = sample->GetEndTime();
1739
if (sampleEndTime > aTrackData.mLastParsedEndTime) {
1740
aTrackData.mLastParsedEndTime = sampleEndTime;
1741
}
1742
1743
// We perform step 10 right away as we can't do anything should a keyframe
1744
// be needed until we have one.
1745
1746
// 10. If the need random access point flag on track buffer equals true,
1747
// then run the following steps:
1748
if (trackBuffer.mNeedRandomAccessPoint) {
1749
// 1. If the coded frame is not a random access point, then drop the coded
1750
// frame and jump to the top of the loop to start processing the next
1751
// coded frame.
1752
if (!sample->mKeyframe) {
1753
previouslyDroppedSample = nullptr;
1754
continue;
1755
}
1756
// 2. Set the need random access point flag on track buffer to false.
1757
trackBuffer.mNeedRandomAccessPoint = false;
1758
}
1759
1760
// We perform step 1,2 and 4 at once:
1761
// 1. If generate timestamps flag equals true:
1762
// Let presentation timestamp equal 0.
1763
// Let decode timestamp equal 0.
1764
// Otherwise:
1765
// Let presentation timestamp be a double precision floating point
1766
// representation of the coded frame's presentation timestamp in seconds.
1767
// Let decode timestamp be a double precision floating point
1768
// representation of the coded frame's decode timestamp in seconds.
1769
1770
// 2. Let frame duration be a double precision floating point representation
1771
// of the coded frame's duration in seconds. Step 3 is performed earlier or
1772
// when a discontinuity has been detected.
1773
// 4. If timestampOffset is not 0, then run the following steps:
1774
1775
TimeUnit sampleTime = sample->mTime;
1776
TimeUnit sampleTimecode = sample->mTimecode;
1777
TimeUnit sampleDuration = sample->mDuration;
1778
TimeUnit timestampOffset = mSourceBufferAttributes->GetTimestampOffset();
1779
1780
TimeInterval sampleInterval =
1781
mSourceBufferAttributes->mGenerateTimestamps
1782
? TimeInterval(timestampOffset, timestampOffset + sampleDuration)
1783
: TimeInterval(timestampOffset + sampleTime,
1784
timestampOffset + sampleTime + sampleDuration);
1785
TimeUnit decodeTimestamp = mSourceBufferAttributes->mGenerateTimestamps
1786
? timestampOffset
1787
: timestampOffset + sampleTimecode;
1788
1789
SAMPLE_DEBUG(
1790
"Processing %s frame [%" PRId64 ",%" PRId64 "] (adjusted:[%" PRId64
1791
",%" PRId64 "]), dts:%" PRId64 ", duration:%" PRId64 ", kf:%d)",
1792
aTrackData.mInfo->mMimeType.get(), sample->mTime.ToMicroseconds(),
1793
sample->GetEndTime().ToMicroseconds(),
1794
sampleInterval.mStart.ToMicroseconds(),
1795
sampleInterval.mEnd.ToMicroseconds(),
1796
sample->mTimecode.ToMicroseconds(), sample->mDuration.ToMicroseconds(),
1797
sample->mKeyframe);
1798
1799
// 6. If last decode timestamp for track buffer is set and decode timestamp
1800
// is less than last decode timestamp: OR If last decode timestamp for track
1801
// buffer is set and the difference between decode timestamp and last decode
1802
// timestamp is greater than 2 times last frame duration:
1803
1804
if (needDiscontinuityCheck && trackBuffer.mLastDecodeTimestamp.isSome() &&
1805
(decodeTimestamp < trackBuffer.mLastDecodeTimestamp.ref() ||
1806
(decodeTimestamp - trackBuffer.mLastDecodeTimestamp.ref() >
1807
trackBuffer.mLongestFrameDuration * 2))) {
1808
MSE_DEBUG("Discontinuity detected.");
1809
SourceBufferAppendMode appendMode =
1810
mSourceBufferAttributes->GetAppendMode();
1811
1812
// 1a. If mode equals "segments":
1813
if (appendMode == SourceBufferAppendMode::Segments) {
1814
// Set group end timestamp to presentation timestamp.
1815
mSourceBufferAttributes->SetGroupEndTimestamp(sampleInterval.mStart);
1816
}
1817
// 1b. If mode equals "sequence":
1818
if (appendMode == SourceBufferAppendMode::Sequence) {
1819
// Set group start timestamp equal to the group end timestamp.
1820
mSourceBufferAttributes->SetGroupStartTimestamp(
1821
mSourceBufferAttributes->GetGroupEndTimestamp());
1822
}
1823
for (auto& track : GetTracksList()) {
1824
// 2. Unset the last decode timestamp on all track buffers.
1825
// 3. Unset the last frame duration on all track buffers.
1826
// 4. Unset the highest end timestamp on all track buffers.
1827
// 5. Set the need random access point flag on all track buffers to
1828
// true.
1829
track->ResetAppendState();
1830
}
1831
// 6. Jump to the Loop Top step above to restart processing of the current
1832
// coded frame. Rather that restarting the process for the frame, we run
1833
// the first steps again instead.
1834
// 3. If mode equals "sequence" and group start timestamp is set, then run
1835
// the following steps:
1836
TimeUnit presentationTimestamp =