Source code

Revision control

Other Tools

1
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2
/* vim:set ts=2 sw=2 sts=2 et cindent: */
3
/* This Source Code Form is subject to the terms of the Mozilla Public
4
* License, v. 2.0. If a copy of the MPL was not distributed with this
5
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7
#include "MediaSourceDemuxer.h"
8
9
#include "MediaSourceUtils.h"
10
#include "OpusDecoder.h"
11
#include "SourceBufferList.h"
12
#include "VorbisDecoder.h"
13
#include "VideoUtils.h"
14
#include "nsPrintfCString.h"
15
16
#include <algorithm>
17
#include <limits>
18
#include <stdint.h>
19
20
namespace mozilla {
21
22
typedef TrackInfo::TrackType TrackType;
23
using media::TimeIntervals;
24
using media::TimeUnit;
25
26
MediaSourceDemuxer::MediaSourceDemuxer(AbstractThread* aAbstractMainThread)
27
: mTaskQueue(new TaskQueue(GetMediaThreadPool(MediaThreadType::PLAYBACK),
28
"MediaSourceDemuxer::mTaskQueue")),
29
mMonitor("MediaSourceDemuxer") {
30
MOZ_ASSERT(NS_IsMainThread());
31
}
32
33
constexpr TimeUnit MediaSourceDemuxer::EOS_FUZZ;
34
35
RefPtr<MediaSourceDemuxer::InitPromise> MediaSourceDemuxer::Init() {
36
RefPtr<MediaSourceDemuxer> self = this;
37
return InvokeAsync(GetTaskQueue(), __func__, [self]() {
38
if (self->ScanSourceBuffersForContent()) {
39
return InitPromise::CreateAndResolve(NS_OK, __func__);
40
}
41
42
RefPtr<InitPromise> p = self->mInitPromise.Ensure(__func__);
43
44
return p;
45
});
46
}
47
48
void MediaSourceDemuxer::AddSizeOfResources(
49
MediaSourceDecoder::ResourceSizes* aSizes) {
50
MOZ_ASSERT(NS_IsMainThread());
51
52
// NB: The track buffers must only be accessed on the TaskQueue.
53
RefPtr<MediaSourceDemuxer> self = this;
54
RefPtr<MediaSourceDecoder::ResourceSizes> sizes = aSizes;
55
nsCOMPtr<nsIRunnable> task = NS_NewRunnableFunction(
56
"MediaSourceDemuxer::AddSizeOfResources", [self, sizes]() {
57
for (const RefPtr<TrackBuffersManager>& manager :
58
self->mSourceBuffers) {
59
manager->AddSizeOfResources(sizes);
60
}
61
});
62
63
nsresult rv = GetTaskQueue()->Dispatch(task.forget());
64
MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
65
Unused << rv;
66
}
67
68
void MediaSourceDemuxer::NotifyInitDataArrived() {
69
RefPtr<MediaSourceDemuxer> self = this;
70
nsCOMPtr<nsIRunnable> task = NS_NewRunnableFunction(
71
"MediaSourceDemuxer::NotifyInitDataArrived", [self]() {
72
if (self->mInitPromise.IsEmpty()) {
73
return;
74
}
75
if (self->ScanSourceBuffersForContent()) {
76
self->mInitPromise.ResolveIfExists(NS_OK, __func__);
77
}
78
});
79
nsresult rv = GetTaskQueue()->Dispatch(task.forget());
80
MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
81
Unused << rv;
82
}
83
84
bool MediaSourceDemuxer::ScanSourceBuffersForContent() {
85
MOZ_ASSERT(OnTaskQueue());
86
87
if (mSourceBuffers.IsEmpty()) {
88
return false;
89
}
90
91
MonitorAutoLock mon(mMonitor);
92
93
bool haveEmptySourceBuffer = false;
94
for (const auto& sourceBuffer : mSourceBuffers) {
95
MediaInfo info = sourceBuffer->GetMetadata();
96
if (!info.HasAudio() && !info.HasVideo()) {
97
haveEmptySourceBuffer = true;
98
}
99
if (info.HasAudio() && !mAudioTrack) {
100
mInfo.mAudio = info.mAudio;
101
mAudioTrack = sourceBuffer;
102
}
103
if (info.HasVideo() && !mVideoTrack) {
104
mInfo.mVideo = info.mVideo;
105
mVideoTrack = sourceBuffer;
106
}
107
if (info.IsEncrypted() && !mInfo.IsEncrypted()) {
108
mInfo.mCrypto = info.mCrypto;
109
}
110
}
111
if (mInfo.HasAudio() && mInfo.HasVideo()) {
112
// We have both audio and video. We can ignore non-ready source buffer.
113
return true;
114
}
115
return !haveEmptySourceBuffer;
116
}
117
118
uint32_t MediaSourceDemuxer::GetNumberTracks(TrackType aType) const {
119
MonitorAutoLock mon(mMonitor);
120
121
switch (aType) {
122
case TrackType::kAudioTrack:
123
return mInfo.HasAudio() ? 1u : 0;
124
case TrackType::kVideoTrack:
125
return mInfo.HasVideo() ? 1u : 0;
126
default:
127
return 0;
128
}
129
}
130
131
already_AddRefed<MediaTrackDemuxer> MediaSourceDemuxer::GetTrackDemuxer(
132
TrackType aType, uint32_t aTrackNumber) {
133
RefPtr<TrackBuffersManager> manager = GetManager(aType);
134
if (!manager) {
135
return nullptr;
136
}
137
RefPtr<MediaSourceTrackDemuxer> e =
138
new MediaSourceTrackDemuxer(this, aType, manager);
139
DDLINKCHILD("track demuxer", e.get());
140
mDemuxers.AppendElement(e);
141
return e.forget();
142
}
143
144
bool MediaSourceDemuxer::IsSeekable() const { return true; }
145
146
UniquePtr<EncryptionInfo> MediaSourceDemuxer::GetCrypto() {
147
MonitorAutoLock mon(mMonitor);
148
auto crypto = MakeUnique<EncryptionInfo>();
149
*crypto = mInfo.mCrypto;
150
return crypto;
151
}
152
153
void MediaSourceDemuxer::AttachSourceBuffer(
154
RefPtr<TrackBuffersManager>& aSourceBuffer) {
155
nsCOMPtr<nsIRunnable> task = NewRunnableMethod<RefPtr<TrackBuffersManager>&&>(
156
"MediaSourceDemuxer::DoAttachSourceBuffer", this,
157
&MediaSourceDemuxer::DoAttachSourceBuffer, aSourceBuffer);
158
nsresult rv = GetTaskQueue()->Dispatch(task.forget());
159
MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
160
Unused << rv;
161
}
162
163
void MediaSourceDemuxer::DoAttachSourceBuffer(
164
RefPtr<mozilla::TrackBuffersManager>&& aSourceBuffer) {
165
MOZ_ASSERT(OnTaskQueue());
166
mSourceBuffers.AppendElement(std::move(aSourceBuffer));
167
ScanSourceBuffersForContent();
168
}
169
170
void MediaSourceDemuxer::DetachSourceBuffer(
171
RefPtr<TrackBuffersManager>& aSourceBuffer) {
172
nsCOMPtr<nsIRunnable> task = NewRunnableMethod<RefPtr<TrackBuffersManager>&&>(
173
"MediaSourceDemuxer::DoDetachSourceBuffer", this,
174
&MediaSourceDemuxer::DoDetachSourceBuffer, aSourceBuffer);
175
nsresult rv = GetTaskQueue()->Dispatch(task.forget());
176
MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
177
Unused << rv;
178
}
179
180
void MediaSourceDemuxer::DoDetachSourceBuffer(
181
RefPtr<TrackBuffersManager>&& aSourceBuffer) {
182
MOZ_ASSERT(OnTaskQueue());
183
mSourceBuffers.RemoveElementsBy(
184
[&aSourceBuffer](const RefPtr<TrackBuffersManager> aLinkedSourceBuffer) {
185
return aLinkedSourceBuffer == aSourceBuffer;
186
});
187
{
188
MonitorAutoLock mon(mMonitor);
189
if (aSourceBuffer == mAudioTrack) {
190
mAudioTrack = nullptr;
191
}
192
if (aSourceBuffer == mVideoTrack) {
193
mVideoTrack = nullptr;
194
}
195
}
196
197
for (auto& demuxer : mDemuxers) {
198
if (demuxer->HasManager(aSourceBuffer)) {
199
demuxer->DetachManager();
200
}
201
}
202
ScanSourceBuffersForContent();
203
}
204
205
TrackInfo* MediaSourceDemuxer::GetTrackInfo(TrackType aTrack) {
206
MonitorAutoLock mon(mMonitor);
207
switch (aTrack) {
208
case TrackType::kAudioTrack:
209
return &mInfo.mAudio;
210
case TrackType::kVideoTrack:
211
return &mInfo.mVideo;
212
default:
213
return nullptr;
214
}
215
}
216
217
RefPtr<TrackBuffersManager> MediaSourceDemuxer::GetManager(TrackType aTrack) {
218
MonitorAutoLock mon(mMonitor);
219
switch (aTrack) {
220
case TrackType::kAudioTrack:
221
return mAudioTrack;
222
case TrackType::kVideoTrack:
223
return mVideoTrack;
224
default:
225
return nullptr;
226
}
227
}
228
229
MediaSourceDemuxer::~MediaSourceDemuxer() {
230
mInitPromise.RejectIfExists(NS_ERROR_DOM_MEDIA_CANCELED, __func__);
231
}
232
233
void MediaSourceDemuxer::GetDebugInfo(dom::MediaSourceDemuxerDebugInfo& aInfo) {
234
MonitorAutoLock mon(mMonitor);
235
if (mAudioTrack) {
236
mAudioTrack->GetDebugInfo(aInfo.mAudioTrack);
237
}
238
if (mVideoTrack) {
239
mVideoTrack->GetDebugInfo(aInfo.mVideoTrack);
240
}
241
}
242
243
MediaSourceTrackDemuxer::MediaSourceTrackDemuxer(MediaSourceDemuxer* aParent,
244
TrackInfo::TrackType aType,
245
TrackBuffersManager* aManager)
246
: mParent(aParent),
247
mType(aType),
248
mMonitor("MediaSourceTrackDemuxer"),
249
mManager(aManager),
250
mReset(true),
251
mPreRoll(TimeUnit::FromMicroseconds(
252
OpusDataDecoder::IsOpus(mParent->GetTrackInfo(mType)->mMimeType) ||
253
VorbisDataDecoder::IsVorbis(
254
mParent->GetTrackInfo(mType)->mMimeType)
255
? 80000
256
: mParent->GetTrackInfo(mType)->mMimeType.EqualsLiteral(
257
"audio/mp4a-latm")
258
// AAC encoder delay is by default 2112 audio frames.
259
// See
261
// So we always seek 2112 frames
262
? (2112 * 1000000ULL /
263
mParent->GetTrackInfo(mType)->GetAsAudioInfo()->mRate)
264
: 0)) {}
265
266
UniquePtr<TrackInfo> MediaSourceTrackDemuxer::GetInfo() const {
267
return mParent->GetTrackInfo(mType)->Clone();
268
}
269
270
RefPtr<MediaSourceTrackDemuxer::SeekPromise> MediaSourceTrackDemuxer::Seek(
271
const TimeUnit& aTime) {
272
MOZ_ASSERT(mParent, "Called after BreackCycle()");
273
return InvokeAsync(mParent->GetTaskQueue(), this, __func__,
274
&MediaSourceTrackDemuxer::DoSeek, aTime);
275
}
276
277
RefPtr<MediaSourceTrackDemuxer::SamplesPromise>
278
MediaSourceTrackDemuxer::GetSamples(int32_t aNumSamples) {
279
MOZ_ASSERT(mParent, "Called after BreackCycle()");
280
return InvokeAsync(mParent->GetTaskQueue(), this, __func__,
281
&MediaSourceTrackDemuxer::DoGetSamples, aNumSamples);
282
}
283
284
void MediaSourceTrackDemuxer::Reset() {
285
MOZ_ASSERT(mParent, "Called after BreackCycle()");
286
RefPtr<MediaSourceTrackDemuxer> self = this;
287
nsCOMPtr<nsIRunnable> task =
288
NS_NewRunnableFunction("MediaSourceTrackDemuxer::Reset", [self]() {
289
self->mNextSample.reset();
290
self->mReset = true;
291
if (!self->mManager) {
292
return;
293
}
294
MOZ_ASSERT(self->OnTaskQueue());
295
self->mManager->Seek(self->mType, TimeUnit::Zero(), TimeUnit::Zero());
296
{
297
MonitorAutoLock mon(self->mMonitor);
298
self->mNextRandomAccessPoint =
299
self->mManager->GetNextRandomAccessPoint(
300
self->mType, MediaSourceDemuxer::EOS_FUZZ);
301
}
302
});
303
nsresult rv = mParent->GetTaskQueue()->Dispatch(task.forget());
304
MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
305
Unused << rv;
306
}
307
308
nsresult MediaSourceTrackDemuxer::GetNextRandomAccessPoint(TimeUnit* aTime) {
309
MonitorAutoLock mon(mMonitor);
310
*aTime = mNextRandomAccessPoint;
311
return NS_OK;
312
}
313
314
RefPtr<MediaSourceTrackDemuxer::SkipAccessPointPromise>
315
MediaSourceTrackDemuxer::SkipToNextRandomAccessPoint(
316
const TimeUnit& aTimeThreshold) {
317
return InvokeAsync(mParent->GetTaskQueue(), this, __func__,
318
&MediaSourceTrackDemuxer::DoSkipToNextRandomAccessPoint,
319
aTimeThreshold);
320
}
321
322
media::TimeIntervals MediaSourceTrackDemuxer::GetBuffered() {
323
MonitorAutoLock mon(mMonitor);
324
if (!mManager) {
325
return media::TimeIntervals();
326
}
327
return mManager->Buffered();
328
}
329
330
void MediaSourceTrackDemuxer::BreakCycles() {
331
RefPtr<MediaSourceTrackDemuxer> self = this;
332
nsCOMPtr<nsIRunnable> task =
333
NS_NewRunnableFunction("MediaSourceTrackDemuxer::BreakCycles", [self]() {
334
self->DetachManager();
335
self->mParent = nullptr;
336
});
337
nsresult rv = mParent->GetTaskQueue()->Dispatch(task.forget());
338
MOZ_DIAGNOSTIC_ASSERT(NS_SUCCEEDED(rv));
339
Unused << rv;
340
}
341
342
RefPtr<MediaSourceTrackDemuxer::SeekPromise> MediaSourceTrackDemuxer::DoSeek(
343
const TimeUnit& aTime) {
344
if (!mManager) {
345
return SeekPromise::CreateAndReject(
346
MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
347
RESULT_DETAIL("manager is detached.")),
348
__func__);
349
}
350
351
MOZ_ASSERT(OnTaskQueue());
352
TimeIntervals buffered = mManager->Buffered(mType);
353
// Fuzz factor represents a +/- threshold. So when seeking it allows the gap
354
// to be twice as big as the fuzz value. We only want to allow EOS_FUZZ gap.
355
buffered.SetFuzz(MediaSourceDemuxer::EOS_FUZZ / 2);
356
TimeUnit seekTime = std::max(aTime - mPreRoll, TimeUnit::Zero());
357
358
if (mManager->IsEnded() && seekTime >= buffered.GetEnd()) {
359
// We're attempting to seek past the end time. Cap seekTime so that we seek
360
// to the last sample instead.
361
seekTime = std::max(mManager->HighestStartTime(mType) - mPreRoll,
362
TimeUnit::Zero());
363
}
364
if (!buffered.ContainsWithStrictEnd(seekTime)) {
365
if (!buffered.ContainsWithStrictEnd(aTime)) {
366
// We don't have the data to seek to.
367
return SeekPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA,
368
__func__);
369
}
370
// Theoretically we should reject the promise with WAITING_FOR_DATA,
371
// however, to avoid unwanted regressions we assume that if at this time
372
// we don't have the wanted data it won't come later.
373
// Instead of using the pre-rolled time, use the earliest time available in
374
// the interval.
375
TimeIntervals::IndexType index = buffered.Find(aTime);
376
MOZ_ASSERT(index != TimeIntervals::NoIndex);
377
seekTime = buffered[index].mStart;
378
}
379
seekTime = mManager->Seek(mType, seekTime, MediaSourceDemuxer::EOS_FUZZ);
380
MediaResult result = NS_OK;
381
RefPtr<MediaRawData> sample =
382
mManager->GetSample(mType, TimeUnit::Zero(), result);
383
MOZ_ASSERT(NS_SUCCEEDED(result) && sample);
384
mNextSample = Some(sample);
385
mReset = false;
386
{
387
MonitorAutoLock mon(mMonitor);
388
mNextRandomAccessPoint =
389
mManager->GetNextRandomAccessPoint(mType, MediaSourceDemuxer::EOS_FUZZ);
390
}
391
return SeekPromise::CreateAndResolve(seekTime, __func__);
392
}
393
394
RefPtr<MediaSourceTrackDemuxer::SamplesPromise>
395
MediaSourceTrackDemuxer::DoGetSamples(int32_t aNumSamples) {
396
if (!mManager) {
397
return SamplesPromise::CreateAndReject(
398
MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
399
RESULT_DETAIL("manager is detached.")),
400
__func__);
401
}
402
403
MOZ_ASSERT(OnTaskQueue());
404
if (mReset) {
405
// If a seek (or reset) was recently performed, we ensure that the data
406
// we are about to retrieve is still available.
407
TimeIntervals buffered = mManager->Buffered(mType);
408
buffered.SetFuzz(MediaSourceDemuxer::EOS_FUZZ / 2);
409
410
if (buffered.IsEmpty() && mManager->IsEnded()) {
411
return SamplesPromise::CreateAndReject(NS_ERROR_DOM_MEDIA_END_OF_STREAM,
412
__func__);
413
}
414
if (!buffered.ContainsWithStrictEnd(TimeUnit::Zero())) {
415
return SamplesPromise::CreateAndReject(
416
NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA, __func__);
417
}
418
mReset = false;
419
}
420
RefPtr<MediaRawData> sample;
421
if (mNextSample) {
422
sample = mNextSample.ref();
423
mNextSample.reset();
424
} else {
425
MediaResult result = NS_OK;
426
sample = mManager->GetSample(mType, MediaSourceDemuxer::EOS_FUZZ, result);
427
if (!sample) {
428
if (result == NS_ERROR_DOM_MEDIA_END_OF_STREAM ||
429
result == NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA) {
430
return SamplesPromise::CreateAndReject(
431
(result == NS_ERROR_DOM_MEDIA_END_OF_STREAM && mManager->IsEnded())
432
? NS_ERROR_DOM_MEDIA_END_OF_STREAM
433
: NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA,
434
__func__);
435
}
436
return SamplesPromise::CreateAndReject(result, __func__);
437
}
438
}
439
RefPtr<SamplesHolder> samples = new SamplesHolder;
440
samples->AppendSample(sample);
441
if (mNextRandomAccessPoint <= sample->mTime) {
442
MonitorAutoLock mon(mMonitor);
443
mNextRandomAccessPoint =
444
mManager->GetNextRandomAccessPoint(mType, MediaSourceDemuxer::EOS_FUZZ);
445
}
446
return SamplesPromise::CreateAndResolve(samples, __func__);
447
}
448
449
RefPtr<MediaSourceTrackDemuxer::SkipAccessPointPromise>
450
MediaSourceTrackDemuxer::DoSkipToNextRandomAccessPoint(
451
const TimeUnit& aTimeThreadshold) {
452
if (!mManager) {
453
return SkipAccessPointPromise::CreateAndReject(
454
SkipFailureHolder(MediaResult(NS_ERROR_DOM_MEDIA_FATAL_ERR,
455
RESULT_DETAIL("manager is detached.")),
456
0),
457
__func__);
458
}
459
460
MOZ_ASSERT(OnTaskQueue());
461
uint32_t parsed = 0;
462
// Ensure that the data we are about to skip to is still available.
463
TimeIntervals buffered = mManager->Buffered(mType);
464
buffered.SetFuzz(MediaSourceDemuxer::EOS_FUZZ / 2);
465
if (buffered.ContainsWithStrictEnd(aTimeThreadshold)) {
466
bool found;
467
parsed = mManager->SkipToNextRandomAccessPoint(
468
mType, aTimeThreadshold, MediaSourceDemuxer::EOS_FUZZ, found);
469
if (found) {
470
return SkipAccessPointPromise::CreateAndResolve(parsed, __func__);
471
}
472
}
473
SkipFailureHolder holder(mManager->IsEnded()
474
? NS_ERROR_DOM_MEDIA_END_OF_STREAM
475
: NS_ERROR_DOM_MEDIA_WAITING_FOR_DATA,
476
parsed);
477
return SkipAccessPointPromise::CreateAndReject(holder, __func__);
478
}
479
480
bool MediaSourceTrackDemuxer::HasManager(TrackBuffersManager* aManager) const {
481
MOZ_ASSERT(OnTaskQueue());
482
return mManager == aManager;
483
}
484
485
void MediaSourceTrackDemuxer::DetachManager() {
486
MOZ_ASSERT(OnTaskQueue());
487
MonitorAutoLock mon(mMonitor);
488
mManager = nullptr;
489
}
490
491
} // namespace mozilla