Source code

Revision control

Other Tools

1
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
3
/* This Source Code Form is subject to the terms of the Mozilla Public
4
* License, v. 2.0. If a copy of the MPL was not distributed with this
5
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7
#ifndef MOZILLA_TRACKBUFFERSMANAGER_H_
8
#define MOZILLA_TRACKBUFFERSMANAGER_H_
9
10
#include "mozilla/Atomics.h"
11
#include "mozilla/Maybe.h"
12
#include "mozilla/Mutex.h"
13
#include "mozilla/NotNull.h"
14
#include "mozilla/TaskQueue.h"
15
#include "mozilla/dom/MediaDebugInfoBinding.h"
16
17
#include "MediaContainerType.h"
18
#include "MediaData.h"
19
#include "MediaDataDemuxer.h"
20
#include "MediaResult.h"
21
#include "MediaSourceDecoder.h"
22
#include "MediaSpan.h"
23
#include "SourceBufferTask.h"
24
#include "TimeUnits.h"
25
#include "nsTArray.h"
26
27
namespace mozilla {
28
29
class AbstractThread;
30
class ContainerParser;
31
class MediaByteBuffer;
32
class MediaRawData;
33
class MediaSourceDemuxer;
34
class SourceBufferResource;
35
36
class SourceBufferTaskQueue {
37
public:
38
SourceBufferTaskQueue() {}
39
40
~SourceBufferTaskQueue() {
41
MOZ_ASSERT(mQueue.IsEmpty(), "All tasks must have been processed");
42
}
43
44
void Push(SourceBufferTask* aTask) { mQueue.AppendElement(aTask); }
45
46
already_AddRefed<SourceBufferTask> Pop() {
47
if (!mQueue.Length()) {
48
return nullptr;
49
}
50
RefPtr<SourceBufferTask> task = std::move(mQueue[0]);
51
mQueue.RemoveElementAt(0);
52
return task.forget();
53
}
54
55
nsTArray<RefPtr<SourceBufferTask>>::size_type Length() const {
56
return mQueue.Length();
57
}
58
59
private:
60
nsTArray<RefPtr<SourceBufferTask>> mQueue;
61
};
62
63
DDLoggedTypeDeclName(TrackBuffersManager);
64
65
class TrackBuffersManager final
66
: public DecoderDoctorLifeLogger<TrackBuffersManager> {
67
public:
68
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(TrackBuffersManager);
69
70
enum class EvictDataResult : int8_t {
71
NO_DATA_EVICTED,
72
CANT_EVICT,
73
BUFFER_FULL,
74
};
75
76
typedef TrackInfo::TrackType TrackType;
77
typedef MediaData::Type MediaType;
78
typedef nsTArray<RefPtr<MediaRawData>> TrackBuffer;
79
typedef SourceBufferTask::AppendPromise AppendPromise;
80
typedef SourceBufferTask::RangeRemovalPromise RangeRemovalPromise;
81
82
// Interface for SourceBuffer
83
TrackBuffersManager(MediaSourceDecoder* aParentDecoder,
84
const MediaContainerType& aType);
85
86
// Queue a task to add data to the end of the input buffer and run the MSE
87
// Buffer Append Algorithm
88
// 3.5.5 Buffer Append Algorithm.
90
RefPtr<AppendPromise> AppendData(already_AddRefed<MediaByteBuffer> aData,
91
const SourceBufferAttributes& aAttributes);
92
93
// Queue a task to abort any pending AppendData.
94
// Does nothing at this stage.
95
void AbortAppendData();
96
97
// Queue a task to run MSE Reset Parser State Algorithm.
98
// 3.5.2 Reset Parser State
99
void ResetParserState(SourceBufferAttributes& aAttributes);
100
101
// Queue a task to run the MSE range removal algorithm.
103
RefPtr<RangeRemovalPromise> RangeRemoval(media::TimeUnit aStart,
104
media::TimeUnit aEnd);
105
106
// Schedule data eviction if necessary as the next call to AppendData will
107
// add aSize bytes.
108
// Eviction is done in two steps, first remove data up to aPlaybackTime
109
// and if still more space is needed remove from the end.
110
EvictDataResult EvictData(const media::TimeUnit& aPlaybackTime,
111
int64_t aSize);
112
113
// Queue a task to run ChangeType
114
void ChangeType(const MediaContainerType& aType);
115
116
// Returns the buffered range currently managed.
117
// This may be called on any thread.
118
// Buffered must conform to
120
media::TimeIntervals Buffered() const;
121
media::TimeUnit HighestStartTime() const;
122
media::TimeUnit HighestEndTime() const;
123
124
// Return the size of the data managed by this SourceBufferContentManager.
125
int64_t GetSize() const;
126
127
// Indicate that the MediaSource parent object got into "ended" state.
128
void Ended();
129
130
// The parent SourceBuffer is about to be destroyed.
131
void Detach();
132
133
int64_t EvictionThreshold() const;
134
135
// Interface for MediaSourceDemuxer
136
MediaInfo GetMetadata() const;
137
const TrackBuffer& GetTrackBuffer(TrackInfo::TrackType aTrack) const;
138
const media::TimeIntervals& Buffered(TrackInfo::TrackType) const;
139
const media::TimeUnit& HighestStartTime(TrackInfo::TrackType) const;
140
media::TimeIntervals SafeBuffered(TrackInfo::TrackType) const;
141
bool IsEnded() const { return mEnded; }
142
uint32_t Evictable(TrackInfo::TrackType aTrack) const;
143
media::TimeUnit Seek(TrackInfo::TrackType aTrack,
144
const media::TimeUnit& aTime,
145
const media::TimeUnit& aFuzz);
146
uint32_t SkipToNextRandomAccessPoint(TrackInfo::TrackType aTrack,
147
const media::TimeUnit& aTimeThreadshold,
148
const media::TimeUnit& aFuzz,
149
bool& aFound);
150
151
already_AddRefed<MediaRawData> GetSample(TrackInfo::TrackType aTrack,
152
const media::TimeUnit& aFuzz,
153
MediaResult& aResult);
154
int32_t FindCurrentPosition(TrackInfo::TrackType aTrack,
155
const media::TimeUnit& aFuzz) const;
156
157
// Will set the next GetSample index if needed. This information is determined
158
// through the value of mNextSampleTimecode. Return false if the index
159
// couldn't be determined or if there's nothing more that could be demuxed.
160
// This occurs if either the track buffer doesn't contain the required
161
// timecode or is empty.
162
nsresult SetNextGetSampleIndexIfNeeded(TrackInfo::TrackType aTrack,
163
const media::TimeUnit& aFuzz);
164
165
media::TimeUnit GetNextRandomAccessPoint(TrackInfo::TrackType aTrack,
166
const media::TimeUnit& aFuzz);
167
168
void AddSizeOfResources(MediaSourceDecoder::ResourceSizes* aSizes) const;
169
void GetDebugInfo(dom::TrackBuffersManagerDebugInfo& aInfo);
170
171
private:
172
typedef MozPromise<bool, MediaResult, /* IsExclusive = */ true>
173
CodedFrameProcessingPromise;
174
175
~TrackBuffersManager();
176
// All following functions run on the taskqueue.
177
RefPtr<AppendPromise> DoAppendData(already_AddRefed<MediaByteBuffer> aData,
178
const SourceBufferAttributes& aAttributes);
179
void ScheduleSegmentParserLoop();
180
void SegmentParserLoop();
181
void InitializationSegmentReceived();
182
void ShutdownDemuxers();
183
void CreateDemuxerforMIMEType();
184
void ResetDemuxingState();
185
void NeedMoreData();
186
void RejectAppend(const MediaResult& aRejectValue, const char* aName);
187
// Will return a promise that will be resolved once all frames of the current
188
// media segment have been processed.
189
RefPtr<CodedFrameProcessingPromise> CodedFrameProcessing();
190
void CompleteCodedFrameProcessing();
191
// Called by ResetParserState.
192
void CompleteResetParserState();
193
RefPtr<RangeRemovalPromise> CodedFrameRemovalWithPromise(
194
media::TimeInterval aInterval);
195
bool CodedFrameRemoval(media::TimeInterval aInterval);
196
// Removes all coded frames -- this is not to spec and should be used as a
197
// last resort to clear buffers only if other methods cannot.
198
void RemoveAllCodedFrames();
199
void SetAppendState(SourceBufferAttributes::AppendState aAppendState);
200
201
bool HasVideo() const { return mVideoTracks.mNumTracks > 0; }
202
bool HasAudio() const { return mAudioTracks.mNumTracks > 0; }
203
204
// The input buffer as per
206
Maybe<MediaSpan> mInputBuffer;
207
// Buffer full flag as per
209
// on both the main thread and the task queue.
210
Atomic<bool> mBufferFull;
211
bool mFirstInitializationSegmentReceived;
212
bool mChangeTypeReceived;
213
// Set to true once a new segment is started.
214
bool mNewMediaSegmentStarted;
215
bool mActiveTrack;
216
MediaContainerType mType;
217
218
// ContainerParser objects and methods.
219
// Those are used to parse the incoming input buffer.
220
221
// Recreate the ContainerParser and if aReuseInitData is true then
222
// feed it with the previous init segment found.
223
void RecreateParser(bool aReuseInitData);
224
UniquePtr<ContainerParser> mParser;
225
226
// Demuxer objects and methods.
227
void AppendDataToCurrentInputBuffer(const MediaSpan& aData);
228
229
RefPtr<MediaByteBuffer> mInitData;
230
// Temporary input buffer to handle partial media segment header.
231
// We store the current input buffer content into it should we need to
232
// reinitialize the demuxer once we have some samples and a discontinuity is
233
// detected.
234
Maybe<MediaSpan> mPendingInputBuffer;
235
RefPtr<SourceBufferResource> mCurrentInputBuffer;
236
RefPtr<MediaDataDemuxer> mInputDemuxer;
237
// Length already processed in current media segment.
238
uint64_t mProcessedInput;
239
Maybe<media::TimeUnit> mLastParsedEndTime;
240
241
void OnDemuxerInitDone(const MediaResult& aResult);
242
void OnDemuxerInitFailed(const MediaResult& aFailure);
243
void OnDemuxerResetDone(const MediaResult& aResult);
244
MozPromiseRequestHolder<MediaDataDemuxer::InitPromise> mDemuxerInitRequest;
245
246
void OnDemuxFailed(TrackType aTrack, const MediaResult& aError);
247
void DoDemuxVideo();
248
void OnVideoDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples);
249
void OnVideoDemuxFailed(const MediaResult& aError) {
250
mVideoTracks.mDemuxRequest.Complete();
251
OnDemuxFailed(TrackType::kVideoTrack, aError);
252
}
253
void DoDemuxAudio();
254
void OnAudioDemuxCompleted(RefPtr<MediaTrackDemuxer::SamplesHolder> aSamples);
255
void OnAudioDemuxFailed(const MediaResult& aError) {
256
mAudioTracks.mDemuxRequest.Complete();
257
OnDemuxFailed(TrackType::kAudioTrack, aError);
258
}
259
260
// Dispatches an "encrypted" event is any sample in array has initData
261
// present.
262
void MaybeDispatchEncryptedEvent(
263
const nsTArray<RefPtr<MediaRawData>>& aSamples);
264
265
void DoEvictData(const media::TimeUnit& aPlaybackTime, int64_t aSizeToEvict);
266
267
struct TrackData {
268
TrackData() : mNumTracks(0), mNeedRandomAccessPoint(true), mSizeBuffer(0) {}
269
uint32_t mNumTracks;
270
// Definition of variables:
272
// Last decode timestamp variable that stores the decode timestamp of the
273
// last coded frame appended in the current coded frame group.
274
// The variable is initially unset to indicate that no coded frames have
275
// been appended yet.
276
Maybe<media::TimeUnit> mLastDecodeTimestamp;
277
// Last frame duration variable that stores the coded frame duration of the
278
// last coded frame appended in the current coded frame group.
279
// The variable is initially unset to indicate that no coded frames have
280
// been appended yet.
281
Maybe<media::TimeUnit> mLastFrameDuration;
282
// Highest end timestamp variable that stores the highest coded frame end
283
// timestamp across all coded frames in the current coded frame group that
284
// were appended to this track buffer.
285
// The variable is initially unset to indicate that no coded frames have
286
// been appended yet.
287
Maybe<media::TimeUnit> mHighestEndTimestamp;
288
// Highest presentation timestamp in track buffer.
289
// Protected by global monitor, except when reading on the task queue as it
290
// is only written there.
291
media::TimeUnit mHighestStartTimestamp;
292
// Longest frame duration seen since last random access point.
293
// Only ever accessed when mLastDecodeTimestamp and mLastFrameDuration are
294
// set.
295
media::TimeUnit mLongestFrameDuration;
296
// Need random access point flag variable that keeps track of whether the
297
// track buffer is waiting for a random access point coded frame.
298
// The variable is initially set to true to indicate that random access
299
// point coded frame is needed before anything can be added to the track
300
// buffer.
301
bool mNeedRandomAccessPoint;
302
RefPtr<MediaTrackDemuxer> mDemuxer;
303
MozPromiseRequestHolder<MediaTrackDemuxer::SamplesPromise> mDemuxRequest;
304
// Highest end timestamp of the last media segment demuxed.
305
media::TimeUnit mLastParsedEndTime;
306
307
// If set, position where the next contiguous frame will be inserted.
308
// If a discontinuity is detected, it will be unset and recalculated upon
309
// the next insertion.
310
Maybe<uint32_t> mNextInsertionIndex;
311
// Samples just demuxed, but not yet parsed.
312
TrackBuffer mQueuedSamples;
313
const TrackBuffer& GetTrackBuffer() const {
314
MOZ_RELEASE_ASSERT(mBuffers.Length(),
315
"TrackBuffer must have been created");
316
return mBuffers.LastElement();
317
}
318
TrackBuffer& GetTrackBuffer() {
319
MOZ_RELEASE_ASSERT(mBuffers.Length(),
320
"TrackBuffer must have been created");
321
return mBuffers.LastElement();
322
}
323
// We only manage a single track of each type at this time.
324
nsTArray<TrackBuffer> mBuffers;
325
// Track buffer ranges variable that represents the presentation time ranges
326
// occupied by the coded frames currently stored in the track buffer.
327
media::TimeIntervals mBufferedRanges;
328
// Sanitized mBufferedRanges with a fuzz of half a sample's duration applied
329
// This buffered ranges is the basis of what is exposed to the JS.
330
media::TimeIntervals mSanitizedBufferedRanges;
331
// Byte size of all samples contained in this track buffer.
332
uint32_t mSizeBuffer;
333
// TrackInfo of the first metadata received.
334
RefPtr<TrackInfoSharedPtr> mInfo;
335
// TrackInfo of the last metadata parsed (updated with each init segment.
336
RefPtr<TrackInfoSharedPtr> mLastInfo;
337
338
// If set, position of the next sample to be retrieved by GetSample().
339
// If the position is equal to the TrackBuffer's length, it indicates that
340
// we've reached EOS.
341
Maybe<uint32_t> mNextGetSampleIndex;
342
// Approximation of the next sample's decode timestamp.
343
media::TimeUnit mNextSampleTimecode;
344
// Approximation of the next sample's presentation timestamp.
345
media::TimeUnit mNextSampleTime;
346
347
struct EvictionIndex {
348
EvictionIndex() { Reset(); }
349
void Reset() {
350
mEvictable = 0;
351
mLastIndex = 0;
352
}
353
uint32_t mEvictable;
354
uint32_t mLastIndex;
355
};
356
// Size of data that can be safely evicted during the next eviction
357
// cycle.
358
// We consider as evictable all frames up to the last keyframe prior to
359
// mNextGetSampleIndex. If mNextGetSampleIndex isn't set, then we assume
360
// that we can't yet evict data.
361
// Protected by global monitor, except when reading on the task queue as it
362
// is only written there.
363
EvictionIndex mEvictionIndex;
364
365
void ResetAppendState() {
366
mLastDecodeTimestamp.reset();
367
mLastFrameDuration.reset();
368
mHighestEndTimestamp.reset();
369
mNeedRandomAccessPoint = true;
370
mNextInsertionIndex.reset();
371
}
372
373
void Reset() {
374
ResetAppendState();
375
mEvictionIndex.Reset();
376
for (auto& buffer : mBuffers) {
377
buffer.Clear();
378
}
379
mSizeBuffer = 0;
380
mNextGetSampleIndex.reset();
381
mBufferedRanges.Clear();
382
mSanitizedBufferedRanges.Clear();
383
}
384
385
void AddSizeOfResources(MediaSourceDecoder::ResourceSizes* aSizes) const;
386
};
387
388
void CheckSequenceDiscontinuity(const media::TimeUnit& aPresentationTime);
389
void ProcessFrames(TrackBuffer& aSamples, TrackData& aTrackData);
390
media::TimeInterval PresentationInterval(const TrackBuffer& aSamples) const;
391
bool CheckNextInsertionIndex(TrackData& aTrackData,
392
const media::TimeUnit& aSampleTime);
393
void InsertFrames(TrackBuffer& aSamples,
394
const media::TimeIntervals& aIntervals,
395
TrackData& aTrackData);
396
void UpdateHighestTimestamp(TrackData& aTrackData,
397
const media::TimeUnit& aHighestTime);
398
// Remove all frames and their dependencies contained in aIntervals.
399
// Return the index at which frames were first removed or 0 if no frames
400
// removed.
401
enum class RemovalMode {
402
kRemoveFrame,
403
kTruncateFrame,
404
};
405
uint32_t RemoveFrames(const media::TimeIntervals& aIntervals,
406
TrackData& aTrackData, uint32_t aStartIndex,
407
RemovalMode aMode);
408
// Recalculate track's evictable amount.
409
void ResetEvictionIndex(TrackData& aTrackData);
410
void UpdateEvictionIndex(TrackData& aTrackData, uint32_t aCurrentIndex);
411
// Find index of sample. Return a negative value if not found.
412
uint32_t FindSampleIndex(const TrackBuffer& aTrackBuffer,
413
const media::TimeInterval& aInterval);
414
const MediaRawData* GetSample(TrackInfo::TrackType aTrack, uint32_t aIndex,
415
const media::TimeUnit& aExpectedDts,
416
const media::TimeUnit& aExpectedPts,
417
const media::TimeUnit& aFuzz);
418
void UpdateBufferedRanges();
419
void RejectProcessing(const MediaResult& aRejectValue, const char* aName);
420
void ResolveProcessing(bool aResolveValue, const char* aName);
421
MozPromiseRequestHolder<CodedFrameProcessingPromise> mProcessingRequest;
422
MozPromiseHolder<CodedFrameProcessingPromise> mProcessingPromise;
423
424
// Trackbuffers definition.
425
nsTArray<const TrackData*> GetTracksList() const;
426
nsTArray<TrackData*> GetTracksList();
427
TrackData& GetTracksData(TrackType aTrack) {
428
switch (aTrack) {
429
case TrackType::kVideoTrack:
430
return mVideoTracks;
431
case TrackType::kAudioTrack:
432
default:
433
return mAudioTracks;
434
}
435
}
436
const TrackData& GetTracksData(TrackType aTrack) const {
437
switch (aTrack) {
438
case TrackType::kVideoTrack:
439
return mVideoTracks;
440
case TrackType::kAudioTrack:
441
default:
442
return mAudioTracks;
443
}
444
}
445
TrackData mVideoTracks;
446
TrackData mAudioTracks;
447
448
// TaskQueue methods and objects.
449
RefPtr<TaskQueue> GetTaskQueueSafe() const {
450
MutexAutoLock mut(mMutex);
451
return mTaskQueue;
452
}
453
NotNull<AbstractThread*> TaskQueueFromTaskQueue() const {
454
#ifdef DEBUG
455
RefPtr<TaskQueue> taskQueue = GetTaskQueueSafe();
456
MOZ_ASSERT(taskQueue && taskQueue->IsCurrentThreadIn());
457
#endif
458
return WrapNotNull(mTaskQueue.get());
459
}
460
bool OnTaskQueue() const {
461
auto taskQueue = TaskQueueFromTaskQueue();
462
return taskQueue->IsCurrentThreadIn();
463
}
464
void ResetTaskQueue() {
465
MutexAutoLock mut(mMutex);
466
mTaskQueue = nullptr;
467
}
468
469
// SourceBuffer Queues and running context.
470
SourceBufferTaskQueue mQueue;
471
void QueueTask(SourceBufferTask* aTask);
472
void ProcessTasks();
473
// Set if the TrackBuffersManager is currently processing a task.
474
// At this stage, this task is always a AppendBufferTask.
475
RefPtr<SourceBufferTask> mCurrentTask;
476
// Current SourceBuffer state for ongoing task.
477
// Its content is returned to the SourceBuffer once the AppendBufferTask has
478
// completed.
479
UniquePtr<SourceBufferAttributes> mSourceBufferAttributes;
480
// The current sourcebuffer append window. It's content is equivalent to
481
// mSourceBufferAttributes.mAppendWindowStart/End
482
media::TimeInterval mAppendWindow;
483
484
// Strong references to external objects.
485
nsMainThreadPtrHandle<MediaSourceDecoder> mParentDecoder;
486
487
const RefPtr<AbstractThread> mAbstractMainThread;
488
489
// Return public highest end time across all aTracks.
490
// Monitor must be held.
491
media::TimeUnit HighestEndTime(
492
nsTArray<const media::TimeIntervals*>& aTracks) const;
493
494
// Set to true if mediasource state changed to ended.
495
Atomic<bool> mEnded;
496
497
// Global size of this source buffer content.
498
Atomic<int64_t> mSizeSourceBuffer;
499
const int64_t mVideoEvictionThreshold;
500
const int64_t mAudioEvictionThreshold;
501
enum class EvictionState {
502
NO_EVICTION_NEEDED,
503
EVICTION_NEEDED,
504
EVICTION_COMPLETED,
505
};
506
Atomic<EvictionState> mEvictionState;
507
508
// Monitor to protect following objects accessed across multiple threads.
509
mutable Mutex mMutex;
510
// mTaskQueue is only ever written after construction on the task queue.
511
// As such, it can be accessed while on task queue without the need for the
512
// mutex.
513
RefPtr<TaskQueue> mTaskQueue;
514
// Stable audio and video track time ranges.
515
media::TimeIntervals mVideoBufferedRanges;
516
media::TimeIntervals mAudioBufferedRanges;
517
// MediaInfo of the first init segment read.
518
MediaInfo mInfo;
519
};
520
521
} // namespace mozilla
522
523
#endif /* MOZILLA_TRACKBUFFERSMANAGER_H_ */