Source code

Revision control

Other Tools

1
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2
* This Source Code Form is subject to the terms of the Mozilla Public
3
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
4
* You can obtain one at http://mozilla.org/MPL/2.0/. */
5
6
#include "MediaEngineRemoteVideoSource.h"
7
8
#include "CamerasChild.h"
9
#include "MediaManager.h"
10
#include "MediaTrackConstraints.h"
11
#include "mozilla/ErrorNames.h"
12
#include "mozilla/RefPtr.h"
13
#include "nsIPrefService.h"
14
#include "Tracing.h"
15
#include "VideoFrameUtils.h"
16
#include "VideoUtils.h"
17
#include "webrtc/common_video/include/video_frame_buffer.h"
18
#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
19
20
namespace mozilla {
21
22
extern LazyLogModule gMediaManagerLog;
23
#define LOG(...) MOZ_LOG(gMediaManagerLog, LogLevel::Debug, (__VA_ARGS__))
24
#define LOG_FRAME(...) \
25
MOZ_LOG(gMediaManagerLog, LogLevel::Verbose, (__VA_ARGS__))
26
27
using dom::ConstrainLongRange;
28
using dom::MediaSourceEnum;
29
using dom::MediaTrackConstraints;
30
using dom::MediaTrackConstraintSet;
31
using dom::MediaTrackSettings;
32
using dom::VideoFacingModeEnum;
33
34
static Maybe<VideoFacingModeEnum> GetFacingMode(const nsString& aDeviceName) {
35
// Set facing mode based on device name.
36
#if defined(ANDROID)
37
// Names are generated. Example: "Camera 0, Facing back, Orientation 90"
38
//
39
// See media/webrtc/trunk/webrtc/modules/video_capture/android/java/src/org/
40
// webrtc/videoengine/VideoCaptureDeviceInfoAndroid.java
41
42
if (aDeviceName.Find(NS_LITERAL_STRING("Facing back")) != kNotFound) {
43
return Some(VideoFacingModeEnum::Environment);
44
}
45
if (aDeviceName.Find(NS_LITERAL_STRING("Facing front")) != kNotFound) {
46
return Some(VideoFacingModeEnum::User);
47
}
48
#endif // ANDROID
49
#ifdef XP_MACOSX
50
// Kludge to test user-facing cameras on OSX.
51
if (aDeviceName.Find(NS_LITERAL_STRING("Face")) != -1) {
52
return Some(VideoFacingModeEnum::User);
53
}
54
#endif
55
#ifdef XP_WIN
56
// The cameras' name of Surface book are "Microsoft Camera Front" and
57
// "Microsoft Camera Rear" respectively.
58
59
if (aDeviceName.Find(NS_LITERAL_STRING("Front")) != kNotFound) {
60
return Some(VideoFacingModeEnum::User);
61
}
62
if (aDeviceName.Find(NS_LITERAL_STRING("Rear")) != kNotFound) {
63
return Some(VideoFacingModeEnum::Environment);
64
}
65
#endif // WINDOWS
66
67
return Nothing();
68
}
69
70
MediaEngineRemoteVideoSource::MediaEngineRemoteVideoSource(
71
int aIndex, camera::CaptureEngine aCapEngine, bool aScary)
72
: mCaptureIndex(aIndex),
73
mCapEngine(aCapEngine),
74
mScary(aScary),
75
mMutex("MediaEngineRemoteVideoSource::mMutex"),
76
mRescalingBufferPool(/* zero_initialize */ false,
77
/* max_number_of_buffers */ 1),
78
mSettingsUpdatedByFrame(MakeAndAddRef<media::Refcountable<AtomicBool>>()),
79
mSettings(MakeAndAddRef<media::Refcountable<MediaTrackSettings>>()),
80
mFirstFramePromise(mFirstFramePromiseHolder.Ensure(__func__)) {
81
mSettings->mWidth.Construct(0);
82
mSettings->mHeight.Construct(0);
83
mSettings->mFrameRate.Construct(0);
84
Init();
85
}
86
87
MediaEngineRemoteVideoSource::~MediaEngineRemoteVideoSource() {
88
mFirstFramePromiseHolder.RejectIfExists(NS_ERROR_ABORT, __func__);
89
}
90
91
dom::MediaSourceEnum MediaEngineRemoteVideoSource::GetMediaSource() const {
92
switch (mCapEngine) {
93
case camera::BrowserEngine:
94
return MediaSourceEnum::Browser;
95
case camera::CameraEngine:
96
return MediaSourceEnum::Camera;
97
case camera::ScreenEngine:
98
return MediaSourceEnum::Screen;
99
case camera::WinEngine:
100
return MediaSourceEnum::Window;
101
default:
102
MOZ_CRASH();
103
}
104
}
105
106
void MediaEngineRemoteVideoSource::Init() {
107
LOG(__PRETTY_FUNCTION__);
108
AssertIsOnOwningThread();
109
110
char deviceName[kMaxDeviceNameLength];
111
char uniqueId[kMaxUniqueIdLength];
112
if (camera::GetChildAndCall(&camera::CamerasChild::GetCaptureDevice,
113
mCapEngine, mCaptureIndex, deviceName,
114
kMaxDeviceNameLength, uniqueId,
115
kMaxUniqueIdLength, nullptr)) {
116
LOG("Error initializing RemoteVideoSource (GetCaptureDevice)");
117
return;
118
}
119
120
SetName(NS_ConvertUTF8toUTF16(deviceName));
121
SetUUID(uniqueId);
122
SetGroupId(NS_ConvertUTF8toUTF16(deviceName));
123
124
mInitDone = true;
125
}
126
127
void MediaEngineRemoteVideoSource::Shutdown() {
128
LOG(__PRETTY_FUNCTION__);
129
AssertIsOnOwningThread();
130
131
if (!mInitDone) {
132
// Already shut down
133
return;
134
}
135
136
if (mState == kStarted) {
137
Stop();
138
}
139
if (mState == kAllocated || mState == kStopped) {
140
Deallocate();
141
}
142
MOZ_ASSERT(mState == kReleased);
143
144
mInitDone = false;
145
}
146
147
void MediaEngineRemoteVideoSource::SetName(nsString aName) {
148
LOG(__PRETTY_FUNCTION__);
149
AssertIsOnOwningThread();
150
151
mDeviceName = std::move(aName);
152
153
Maybe<VideoFacingModeEnum> facingMode;
154
if (GetMediaSource() == MediaSourceEnum::Camera) {
155
// Only cameras can have a facing mode.
156
facingMode = GetFacingMode(mDeviceName);
157
}
158
159
if (facingMode.isSome()) {
160
mFacingMode.Assign(NS_ConvertUTF8toUTF16(
161
dom::VideoFacingModeEnumValues::strings[uint32_t(*facingMode)].value));
162
} else {
163
mFacingMode.Truncate();
164
}
165
NS_DispatchToMainThread(NS_NewRunnableFunction(
166
"MediaEngineRemoteVideoSource::SetName (facingMode updater)",
167
[settings = mSettings, hasFacingMode = facingMode.isSome(),
168
mode = mFacingMode]() {
169
if (!hasFacingMode) {
170
settings->mFacingMode.Reset();
171
return;
172
}
173
settings->mFacingMode.Construct(mode);
174
}));
175
}
176
177
nsString MediaEngineRemoteVideoSource::GetName() const {
178
AssertIsOnOwningThread();
179
180
return mDeviceName;
181
}
182
183
void MediaEngineRemoteVideoSource::SetUUID(const char* aUUID) {
184
AssertIsOnOwningThread();
185
186
mUniqueId.Assign(aUUID);
187
}
188
189
nsCString MediaEngineRemoteVideoSource::GetUUID() const {
190
AssertIsOnOwningThread();
191
192
return mUniqueId;
193
}
194
195
void MediaEngineRemoteVideoSource::SetGroupId(nsString aGroupId) {
196
AssertIsOnOwningThread();
197
198
mGroupId = std::move(aGroupId);
199
}
200
201
nsString MediaEngineRemoteVideoSource::GetGroupId() const {
202
AssertIsOnOwningThread();
203
204
return mGroupId;
205
}
206
207
nsresult MediaEngineRemoteVideoSource::Allocate(
208
const MediaTrackConstraints& aConstraints, const MediaEnginePrefs& aPrefs,
209
const nsString& aDeviceId,
210
const mozilla::ipc::PrincipalInfo& aPrincipalInfo,
211
const char** aOutBadConstraint) {
212
LOG(__PRETTY_FUNCTION__);
213
AssertIsOnOwningThread();
214
215
MOZ_ASSERT(mState == kReleased);
216
217
if (!mInitDone) {
218
LOG("Init not done");
219
return NS_ERROR_FAILURE;
220
}
221
222
NormalizedConstraints constraints(aConstraints);
223
webrtc::CaptureCapability newCapability;
224
LOG("ChooseCapability(kFitness) for mCapability (Allocate) ++");
225
if (!ChooseCapability(constraints, aPrefs, aDeviceId, newCapability,
226
kFitness)) {
227
*aOutBadConstraint =
228
MediaConstraintsHelper::FindBadConstraint(constraints, this, aDeviceId);
229
return NS_ERROR_FAILURE;
230
}
231
LOG("ChooseCapability(kFitness) for mCapability (Allocate) --");
232
233
if (camera::GetChildAndCall(&camera::CamerasChild::AllocateCaptureDevice,
234
mCapEngine, mUniqueId.get(), kMaxUniqueIdLength,
235
mCaptureIndex, aPrincipalInfo)) {
236
return NS_ERROR_FAILURE;
237
}
238
239
{
240
MutexAutoLock lock(mMutex);
241
mState = kAllocated;
242
mCapability = newCapability;
243
}
244
245
LOG("Video device %d allocated", mCaptureIndex);
246
return NS_OK;
247
}
248
249
nsresult MediaEngineRemoteVideoSource::Deallocate() {
250
LOG(__PRETTY_FUNCTION__);
251
AssertIsOnOwningThread();
252
253
MOZ_ASSERT(mState == kStopped || mState == kAllocated);
254
255
if (mStream && IsTrackIDExplicit(mTrackID)) {
256
mStream->EndTrack(mTrackID);
257
}
258
259
{
260
MutexAutoLock lock(mMutex);
261
262
mStream = nullptr;
263
mTrackID = TRACK_NONE;
264
mPrincipal = PRINCIPAL_HANDLE_NONE;
265
mState = kReleased;
266
}
267
268
// Stop() has stopped capture synchronously on the media thread before we get
269
// here, so there are no longer any callbacks on an IPC thread accessing
270
// mImageContainer or mRescalingBufferPool.
271
mImageContainer = nullptr;
272
mRescalingBufferPool.Release();
273
274
LOG("Video device %d deallocated", mCaptureIndex);
275
276
if (camera::GetChildAndCall(&camera::CamerasChild::ReleaseCaptureDevice,
277
mCapEngine, mCaptureIndex)) {
278
MOZ_ASSERT_UNREACHABLE("Couldn't release allocated device");
279
}
280
return NS_OK;
281
}
282
283
void MediaEngineRemoteVideoSource::SetTrack(
284
const RefPtr<SourceMediaStream>& aStream, TrackID aTrackID,
285
const PrincipalHandle& aPrincipal) {
286
LOG(__PRETTY_FUNCTION__);
287
AssertIsOnOwningThread();
288
289
MOZ_ASSERT(mState == kAllocated);
290
MOZ_ASSERT(!mStream);
291
MOZ_ASSERT(mTrackID == TRACK_NONE);
292
MOZ_ASSERT(aStream);
293
MOZ_ASSERT(IsTrackIDExplicit(aTrackID));
294
295
if (!mImageContainer) {
296
mImageContainer = layers::LayerManager::CreateImageContainer(
297
layers::ImageContainer::ASYNCHRONOUS);
298
}
299
300
{
301
MutexAutoLock lock(mMutex);
302
mStream = aStream;
303
mTrackID = aTrackID;
304
mPrincipal = aPrincipal;
305
}
306
aStream->AddTrack(aTrackID, new VideoSegment(),
307
SourceMediaStream::ADDTRACK_QUEUED);
308
}
309
310
nsresult MediaEngineRemoteVideoSource::Start() {
311
LOG(__PRETTY_FUNCTION__);
312
AssertIsOnOwningThread();
313
314
MOZ_ASSERT(mState == kAllocated || mState == kStopped);
315
MOZ_ASSERT(mInitDone);
316
MOZ_ASSERT(mStream);
317
MOZ_ASSERT(IsTrackIDExplicit(mTrackID));
318
319
{
320
MutexAutoLock lock(mMutex);
321
mState = kStarted;
322
}
323
324
mSettingsUpdatedByFrame->mValue = false;
325
326
if (camera::GetChildAndCall(&camera::CamerasChild::StartCapture, mCapEngine,
327
mCaptureIndex, mCapability, this)) {
328
LOG("StartCapture failed");
329
MutexAutoLock lock(mMutex);
330
mState = kStopped;
331
return NS_ERROR_FAILURE;
332
}
333
334
NS_DispatchToMainThread(NS_NewRunnableFunction(
335
"MediaEngineRemoteVideoSource::SetLastCapability",
336
[settings = mSettings, updated = mSettingsUpdatedByFrame,
337
capEngine = mCapEngine, cap = mCapability]() mutable {
338
switch (capEngine) {
339
case camera::ScreenEngine:
340
case camera::WinEngine:
341
// Undo the hack where ideal and max constraints are crammed
342
// together in mCapability for consumption by low-level code. We
343
// don't actually know the real resolution yet, so report min(ideal,
344
// max) for now.
345
// TODO: This can be removed in bug 1453269.
346
cap.width = std::min(cap.width >> 16, cap.width & 0xffff);
347
cap.height = std::min(cap.height >> 16, cap.height & 0xffff);
348
break;
349
default:
350
break;
351
}
352
353
if (!updated->mValue) {
354
settings->mWidth.Value() = cap.width;
355
settings->mHeight.Value() = cap.height;
356
}
357
settings->mFrameRate.Value() = cap.maxFPS;
358
}));
359
360
return NS_OK;
361
}
362
363
nsresult MediaEngineRemoteVideoSource::FocusOnSelectedSource() {
364
LOG(__PRETTY_FUNCTION__);
365
AssertIsOnOwningThread();
366
367
int result;
368
result = camera::GetChildAndCall(&camera::CamerasChild::FocusOnSelectedSource,
369
mCapEngine, mCaptureIndex);
370
return result == 0 ? NS_OK : NS_ERROR_FAILURE;
371
}
372
373
nsresult MediaEngineRemoteVideoSource::Stop() {
374
LOG(__PRETTY_FUNCTION__);
375
AssertIsOnOwningThread();
376
377
if (mState == kStopped || mState == kAllocated) {
378
return NS_OK;
379
}
380
381
MOZ_ASSERT(mState == kStarted);
382
383
if (camera::GetChildAndCall(&camera::CamerasChild::StopCapture, mCapEngine,
384
mCaptureIndex)) {
385
MOZ_DIAGNOSTIC_ASSERT(false, "Stopping a started capture failed");
386
return NS_ERROR_FAILURE;
387
}
388
389
{
390
MutexAutoLock lock(mMutex);
391
mState = kStopped;
392
}
393
394
return NS_OK;
395
}
396
397
nsresult MediaEngineRemoteVideoSource::Reconfigure(
398
const MediaTrackConstraints& aConstraints, const MediaEnginePrefs& aPrefs,
399
const nsString& aDeviceId, const char** aOutBadConstraint) {
400
LOG(__PRETTY_FUNCTION__);
401
AssertIsOnOwningThread();
402
403
MOZ_ASSERT(mInitDone);
404
405
NormalizedConstraints constraints(aConstraints);
406
webrtc::CaptureCapability newCapability;
407
LOG("ChooseCapability(kFitness) for mTargetCapability (Reconfigure) ++");
408
if (!ChooseCapability(constraints, aPrefs, aDeviceId, newCapability,
409
kFitness)) {
410
*aOutBadConstraint =
411
MediaConstraintsHelper::FindBadConstraint(constraints, this, aDeviceId);
412
return NS_ERROR_INVALID_ARG;
413
}
414
LOG("ChooseCapability(kFitness) for mTargetCapability (Reconfigure) --");
415
416
if (mCapability == newCapability) {
417
return NS_OK;
418
}
419
420
bool started = mState == kStarted;
421
if (started) {
422
nsresult rv = Stop();
423
if (NS_WARN_IF(NS_FAILED(rv))) {
424
nsAutoCString name;
425
GetErrorName(rv, name);
426
LOG("Video source %p for video device %d Reconfigure() failed "
427
"unexpectedly in Stop(). rv=%s",
428
this, mCaptureIndex, name.Data());
429
return NS_ERROR_UNEXPECTED;
430
}
431
}
432
433
{
434
MutexAutoLock lock(mMutex);
435
// Start() applies mCapability on the device.
436
mCapability = newCapability;
437
}
438
439
if (started) {
440
nsresult rv = Start();
441
if (NS_WARN_IF(NS_FAILED(rv))) {
442
nsAutoCString name;
443
GetErrorName(rv, name);
444
LOG("Video source %p for video device %d Reconfigure() failed "
445
"unexpectedly in Start(). rv=%s",
446
this, mCaptureIndex, name.Data());
447
return NS_ERROR_UNEXPECTED;
448
}
449
}
450
451
return NS_OK;
452
}
453
454
size_t MediaEngineRemoteVideoSource::NumCapabilities() const {
455
AssertIsOnOwningThread();
456
457
mHardcodedCapabilities.Clear();
458
int num = camera::GetChildAndCall(&camera::CamerasChild::NumberOfCapabilities,
459
mCapEngine, mUniqueId.get());
460
461
if (num >= 1) {
462
return num;
463
}
464
465
// The default for devices that don't return discrete capabilities: treat
466
// them as supporting all capabilities orthogonally. E.g. screensharing.
467
// CaptureCapability defaults key values to 0, which means accept any value.
468
mHardcodedCapabilities.AppendElement(webrtc::CaptureCapability());
469
return mHardcodedCapabilities.Length(); // 1
470
}
471
472
webrtc::CaptureCapability MediaEngineRemoteVideoSource::GetCapability(
473
size_t aIndex) const {
474
AssertIsOnOwningThread();
475
webrtc::CaptureCapability result;
476
if (!mHardcodedCapabilities.IsEmpty()) {
477
MOZ_ASSERT(aIndex < mHardcodedCapabilities.Length());
478
result = mHardcodedCapabilities.SafeElementAt(aIndex,
479
webrtc::CaptureCapability());
480
}
481
camera::GetChildAndCall(&camera::CamerasChild::GetCaptureCapability,
482
mCapEngine, mUniqueId.get(), aIndex, result);
483
return result;
484
}
485
486
int MediaEngineRemoteVideoSource::DeliverFrame(
487
uint8_t* aBuffer, const camera::VideoFrameProperties& aProps) {
488
// Cameras IPC thread - take great care with accessing members!
489
490
int32_t req_max_width;
491
int32_t req_max_height;
492
int32_t req_ideal_width;
493
int32_t req_ideal_height;
494
{
495
MutexAutoLock lock(mMutex);
496
MOZ_ASSERT(mState == kStarted);
497
// TODO: These can be removed in bug 1453269.
498
req_max_width = mCapability.width & 0xffff;
499
req_max_height = mCapability.height & 0xffff;
500
req_ideal_width = (mCapability.width >> 16) & 0xffff;
501
req_ideal_height = (mCapability.height >> 16) & 0xffff;
502
}
503
504
// This is only used in the case of screen sharing, see bug 1453269.
505
const int32_t target_width = aProps.width();
506
const int32_t target_height = aProps.height();
507
508
if (aProps.rotation() == 90 || aProps.rotation() == 270) {
509
// This frame is rotated, so what was negotiated as width is now height,
510
// and vice versa.
511
std::swap(req_max_width, req_max_height);
512
std::swap(req_ideal_width, req_ideal_height);
513
}
514
515
int32_t dst_max_width = req_max_width == 0
516
? aProps.width()
517
: std::min(req_max_width, aProps.width());
518
int32_t dst_max_height = req_max_height == 0
519
? aProps.height()
520
: std::min(req_max_height, aProps.height());
521
// This logic works for both camera and screen sharing case.
522
// for camera case, req_ideal_width and req_ideal_height is 0.
523
// The following snippet will set dst_width to dst_max_width and dst_height to
524
// dst_max_height
525
int32_t dst_width = std::min(
526
req_ideal_width > 0 ? req_ideal_width : aProps.width(), dst_max_width);
527
int32_t dst_height =
528
std::min(req_ideal_height > 0 ? req_ideal_height : aProps.height(),
529
dst_max_height);
530
531
// Apply scaling for screen sharing, see bug 1453269.
532
switch (mCapEngine) {
533
case camera::ScreenEngine:
534
case camera::WinEngine: {
535
// scale to average of portrait and landscape
536
float scale_width = (float)dst_width / (float)aProps.width();
537
float scale_height = (float)dst_height / (float)aProps.height();
538
float scale = (scale_width + scale_height) / 2;
539
dst_width = (int)(scale * target_width);
540
dst_height = (int)(scale * target_height);
541
542
// if scaled rectangle exceeds max rectangle, scale to minimum of portrait
543
// and landscape
544
if (dst_width > dst_max_width || dst_height > dst_max_height) {
545
scale_width = (float)dst_max_width / (float)dst_width;
546
scale_height = (float)dst_max_height / (float)dst_height;
547
scale = std::min(scale_width, scale_height);
548
dst_width = (int32_t)(scale * dst_width);
549
dst_height = (int32_t)(scale * dst_height);
550
}
551
break;
552
}
553
default: {
554
break;
555
}
556
}
557
558
rtc::Callback0<void> callback_unused;
559
rtc::scoped_refptr<webrtc::I420BufferInterface> buffer =
560
new rtc::RefCountedObject<webrtc::WrappedI420Buffer>(
561
aProps.width(), aProps.height(), aBuffer, aProps.yStride(),
562
aBuffer + aProps.yAllocatedSize(), aProps.uStride(),
563
aBuffer + aProps.yAllocatedSize() + aProps.uAllocatedSize(),
564
aProps.vStride(), callback_unused);
565
566
if ((dst_width != aProps.width() || dst_height != aProps.height()) &&
567
dst_width <= aProps.width() && dst_height <= aProps.height()) {
568
// Destination resolution is smaller than source buffer. We'll rescale.
569
rtc::scoped_refptr<webrtc::I420Buffer> scaledBuffer =
570
mRescalingBufferPool.CreateBuffer(dst_width, dst_height);
571
if (!scaledBuffer) {
572
MOZ_ASSERT_UNREACHABLE(
573
"We might fail to allocate a buffer, but with this "
574
"being a recycling pool that shouldn't happen");
575
return 0;
576
}
577
scaledBuffer->CropAndScaleFrom(*buffer);
578
buffer = scaledBuffer;
579
}
580
581
layers::PlanarYCbCrData data;
582
data.mYChannel = const_cast<uint8_t*>(buffer->DataY());
583
data.mYSize = IntSize(buffer->width(), buffer->height());
584
data.mYStride = buffer->StrideY();
585
MOZ_ASSERT(buffer->StrideU() == buffer->StrideV());
586
data.mCbCrStride = buffer->StrideU();
587
data.mCbChannel = const_cast<uint8_t*>(buffer->DataU());
588
data.mCrChannel = const_cast<uint8_t*>(buffer->DataV());
589
data.mCbCrSize =
590
IntSize((buffer->width() + 1) / 2, (buffer->height() + 1) / 2);
591
data.mPicX = 0;
592
data.mPicY = 0;
593
data.mPicSize = IntSize(buffer->width(), buffer->height());
594
595
RefPtr<layers::PlanarYCbCrImage> image =
596
mImageContainer->CreatePlanarYCbCrImage();
597
if (!image->CopyData(data)) {
598
MOZ_ASSERT_UNREACHABLE(
599
"We might fail to allocate a buffer, but with this "
600
"being a recycling container that shouldn't happen");
601
return 0;
602
}
603
604
#ifdef DEBUG
605
static uint32_t frame_num = 0;
606
LOG_FRAME(
607
"frame %d (%dx%d)->(%dx%d); rotation %d, timeStamp %u, ntpTimeMs %" PRIu64
608
", renderTimeMs %" PRIu64,
609
frame_num++, aProps.width(), aProps.height(), dst_width, dst_height,
610
aProps.rotation(), aProps.timeStamp(), aProps.ntpTimeMs(),
611
aProps.renderTimeMs());
612
#endif
613
614
if (mImageSize.width != dst_width || mImageSize.height != dst_height) {
615
NS_DispatchToMainThread(NS_NewRunnableFunction(
616
"MediaEngineRemoteVideoSource::FrameSizeChange",
617
[settings = mSettings, updated = mSettingsUpdatedByFrame,
618
holder = std::move(mFirstFramePromiseHolder), dst_width,
619
dst_height]() mutable {
620
settings->mWidth.Value() = dst_width;
621
settings->mHeight.Value() = dst_height;
622
updated->mValue = true;
623
// Since mImageSize was initialized to (0,0), we end up here on the
624
// arrival of the first frame. We resolve the promise representing
625
// arrival of first frame, after correct settings values have been
626
// made available (Resolve() is idempotent if already resolved).
627
holder.ResolveIfExists(true, __func__);
628
}));
629
}
630
631
{
632
MutexAutoLock lock(mMutex);
633
MOZ_ASSERT(mState == kStarted);
634
VideoSegment segment;
635
mImageSize = image->GetSize();
636
segment.AppendFrame(image.forget(), mImageSize, mPrincipal);
637
mStream->AppendToTrack(mTrackID, &segment);
638
}
639
640
return 0;
641
}
642
643
uint32_t MediaEngineRemoteVideoSource::GetDistance(
644
const webrtc::CaptureCapability& aCandidate,
645
const NormalizedConstraintSet& aConstraints, const nsString& aDeviceId,
646
const DistanceCalculation aCalculate) const {
647
if (aCalculate == kFeasibility) {
648
return GetFeasibilityDistance(aCandidate, aConstraints, aDeviceId);
649
}
650
return GetFitnessDistance(aCandidate, aConstraints, aDeviceId);
651
}
652
653
uint32_t MediaEngineRemoteVideoSource::GetFitnessDistance(
654
const webrtc::CaptureCapability& aCandidate,
655
const NormalizedConstraintSet& aConstraints,
656
const nsString& aDeviceId) const {
657
AssertIsOnOwningThread();
658
659
// Treat width|height|frameRate == 0 on capability as "can do any".
660
// This allows for orthogonal capabilities that are not in discrete steps.
661
662
typedef MediaConstraintsHelper H;
663
uint64_t distance =
664
uint64_t(H::FitnessDistance(aDeviceId, aConstraints.mDeviceId)) +
665
uint64_t(H::FitnessDistance(mFacingMode, aConstraints.mFacingMode)) +
666
uint64_t(aCandidate.width ? H::FitnessDistance(int32_t(aCandidate.width),
667
aConstraints.mWidth)
668
: 0) +
669
uint64_t(aCandidate.height
670
? H::FitnessDistance(int32_t(aCandidate.height),
671
aConstraints.mHeight)
672
: 0) +
673
uint64_t(aCandidate.maxFPS ? H::FitnessDistance(double(aCandidate.maxFPS),
674
aConstraints.mFrameRate)
675
: 0);
676
return uint32_t(std::min(distance, uint64_t(UINT32_MAX)));
677
}
678
679
uint32_t MediaEngineRemoteVideoSource::GetFeasibilityDistance(
680
const webrtc::CaptureCapability& aCandidate,
681
const NormalizedConstraintSet& aConstraints,
682
const nsString& aDeviceId) const {
683
AssertIsOnOwningThread();
684
685
// Treat width|height|frameRate == 0 on capability as "can do any".
686
// This allows for orthogonal capabilities that are not in discrete steps.
687
688
typedef MediaConstraintsHelper H;
689
uint64_t distance =
690
uint64_t(H::FitnessDistance(aDeviceId, aConstraints.mDeviceId)) +
691
uint64_t(H::FitnessDistance(mFacingMode, aConstraints.mFacingMode)) +
692
uint64_t(aCandidate.width
693
? H::FeasibilityDistance(int32_t(aCandidate.width),
694
aConstraints.mWidth)
695
: 0) +
696
uint64_t(aCandidate.height
697
? H::FeasibilityDistance(int32_t(aCandidate.height),
698
aConstraints.mHeight)
699
: 0) +
700
uint64_t(aCandidate.maxFPS
701
? H::FeasibilityDistance(double(aCandidate.maxFPS),
702
aConstraints.mFrameRate)
703
: 0);
704
return uint32_t(std::min(distance, uint64_t(UINT32_MAX)));
705
}
706
707
// Find best capability by removing inferiors. May leave >1 of equal distance
708
709
/* static */
710
void MediaEngineRemoteVideoSource::TrimLessFitCandidates(
711
nsTArray<CapabilityCandidate>& aSet) {
712
uint32_t best = UINT32_MAX;
713
for (auto& candidate : aSet) {
714
if (best > candidate.mDistance) {
715
best = candidate.mDistance;
716
}
717
}
718
for (size_t i = 0; i < aSet.Length();) {
719
if (aSet[i].mDistance > best) {
720
aSet.RemoveElementAt(i);
721
} else {
722
++i;
723
}
724
}
725
MOZ_ASSERT(aSet.Length());
726
}
727
728
uint32_t MediaEngineRemoteVideoSource::GetBestFitnessDistance(
729
const nsTArray<const NormalizedConstraintSet*>& aConstraintSets,
730
const nsString& aDeviceId) const {
731
AssertIsOnOwningThread();
732
733
size_t num = NumCapabilities();
734
nsTArray<CapabilityCandidate> candidateSet;
735
for (size_t i = 0; i < num; i++) {
736
candidateSet.AppendElement(CapabilityCandidate(GetCapability(i)));
737
}
738
739
bool first = true;
740
for (const NormalizedConstraintSet* ns : aConstraintSets) {
741
for (size_t i = 0; i < candidateSet.Length();) {
742
auto& candidate = candidateSet[i];
743
uint32_t distance =
744
GetFitnessDistance(candidate.mCapability, *ns, aDeviceId);
745
if (distance == UINT32_MAX) {
746
candidateSet.RemoveElementAt(i);
747
} else {
748
++i;
749
if (first) {
750
candidate.mDistance = distance;
751
}
752
}
753
}
754
first = false;
755
}
756
if (!candidateSet.Length()) {
757
return UINT32_MAX;
758
}
759
TrimLessFitCandidates(candidateSet);
760
return candidateSet[0].mDistance;
761
}
762
763
static void LogCapability(const char* aHeader,
764
const webrtc::CaptureCapability& aCapability,
765
uint32_t aDistance) {
766
static const char* const codec[] = {"VP8", "VP9", "H264",
767
"I420", "RED", "ULPFEC",
768
"Generic codec", "Unknown codec"};
769
770
LOG("%s: %4u x %4u x %2u maxFps, %s. Distance = %" PRIu32, aHeader,
771
aCapability.width, aCapability.height, aCapability.maxFPS,
772
codec[std::min(std::max(uint32_t(0), uint32_t(aCapability.videoType)),
773
uint32_t(sizeof(codec) / sizeof(*codec) - 1))],
774
aDistance);
775
}
776
777
bool MediaEngineRemoteVideoSource::ChooseCapability(
778
const NormalizedConstraints& aConstraints, const MediaEnginePrefs& aPrefs,
779
const nsString& aDeviceId, webrtc::CaptureCapability& aCapability,
780
const DistanceCalculation aCalculate) {
781
LOG(__PRETTY_FUNCTION__);
782
AssertIsOnOwningThread();
783
784
if (MOZ_LOG_TEST(gMediaManagerLog, LogLevel::Debug)) {
785
LOG("ChooseCapability: prefs: %dx%d @%dfps", aPrefs.GetWidth(),
786
aPrefs.GetHeight(), aPrefs.mFPS);
787
MediaConstraintsHelper::LogConstraints(aConstraints);
788
if (!aConstraints.mAdvanced.empty()) {
789
LOG("Advanced array[%zu]:", aConstraints.mAdvanced.size());
790
for (auto& advanced : aConstraints.mAdvanced) {
791
MediaConstraintsHelper::LogConstraints(advanced);
792
}
793
}
794
}
795
796
switch (mCapEngine) {
797
case camera::ScreenEngine:
798
case camera::WinEngine: {
799
FlattenedConstraints c(aConstraints);
800
// The actual resolution to constrain around is not easy to find ahead of
801
// time (and may in fact change over time), so as a hack, we push ideal
802
// and max constraints down to desktop_capture_impl.cc and finish the
803
// algorithm there.
804
// TODO: This can be removed in bug 1453269.
805
aCapability.width =
806
(std::min(0xffff, c.mWidth.mIdeal.valueOr(0)) & 0xffff) << 16 |
807
(std::min(0xffff, c.mWidth.mMax) & 0xffff);
808
aCapability.height =
809
(std::min(0xffff, c.mHeight.mIdeal.valueOr(0)) & 0xffff) << 16 |
810
(std::min(0xffff, c.mHeight.mMax) & 0xffff);
811
aCapability.maxFPS =
812
c.mFrameRate.Clamp(c.mFrameRate.mIdeal.valueOr(aPrefs.mFPS));
813
return true;
814
}
815
default:
816
break;
817
}
818
819
nsTArray<CapabilityCandidate> candidateSet;
820
size_t num = NumCapabilities();
821
for (size_t i = 0; i < num; i++) {
822
candidateSet.AppendElement(CapabilityCandidate(GetCapability(i)));
823
}
824
825
if (!mHardcodedCapabilities.IsEmpty() && mCapEngine == camera::CameraEngine) {
826
// We have a hardcoded capability, which means this camera didn't report
827
// discrete capabilities. It might still allow a ranged capability, so we
828
// add a couple of default candidates based on prefs and constraints.
829
// The chosen candidate will be propagated to StartCapture() which will fail
830
// for an invalid candidate.
831
MOZ_DIAGNOSTIC_ASSERT(mHardcodedCapabilities.Length() == 1);
832
MOZ_DIAGNOSTIC_ASSERT(candidateSet.Length() == 1);
833
candidateSet.Clear();
834
835
FlattenedConstraints c(aConstraints);
836
// Reuse the code across both the low-definition (`false`) pref and
837
// the high-definition (`true`) pref.
838
// If there are constraints we try to satisfy them but we default to prefs.
839
// Note that since constraints are from content and can literally be
840
// anything we put (rather generous) caps on them.
841
for (bool isHd : {false, true}) {
842
webrtc::CaptureCapability cap;
843
int32_t prefWidth = aPrefs.GetWidth(isHd);
844
int32_t prefHeight = aPrefs.GetHeight(isHd);
845
846
cap.width = c.mWidth.Get(prefWidth);
847
cap.width = std::max(0, std::min(cap.width, 7680));
848
849
cap.height = c.mHeight.Get(prefHeight);
850
cap.height = std::max(0, std::min(cap.height, 4320));
851
852
cap.maxFPS = c.mFrameRate.Get(aPrefs.mFPS);
853
cap.maxFPS = std::max(0, std::min(cap.maxFPS, 480));
854
855
if (cap.width != prefWidth) {
856
// Width was affected by constraints.
857
// We'll adjust the height too so the aspect ratio is retained.
858
cap.height = cap.width * prefHeight / prefWidth;
859
} else if (cap.height != prefHeight) {
860
// Height was affected by constraints but not width.
861
// We'll adjust the width too so the aspect ratio is retained.
862
cap.width = cap.height * prefWidth / prefHeight;
863
}
864
865
if (candidateSet.Contains(cap, CapabilityComparator())) {
866
continue;
867
}
868
LogCapability("Hardcoded capability", cap, 0);
869
candidateSet.AppendElement(cap);
870
}
871
}
872
873
// First, filter capabilities by required constraints (min, max, exact).
874
875
for (size_t i = 0; i < candidateSet.Length();) {
876
auto& candidate = candidateSet[i];
877
candidate.mDistance =
878
GetDistance(candidate.mCapability, aConstraints, aDeviceId, aCalculate);
879
LogCapability("Capability", candidate.mCapability, candidate.mDistance);
880
if (candidate.mDistance == UINT32_MAX) {
881
candidateSet.RemoveElementAt(i);
882
} else {
883
++i;
884
}
885
}
886
887
if (candidateSet.IsEmpty()) {
888
LOG("failed to find capability match from %zu choices",
889
candidateSet.Length());
890
return false;
891
}
892
893
// Filter further with all advanced constraints (that don't overconstrain).
894
895
for (const auto& cs : aConstraints.mAdvanced) {
896
nsTArray<CapabilityCandidate> rejects;
897
for (size_t i = 0; i < candidateSet.Length();) {
898
if (GetDistance(candidateSet[i].mCapability, cs, aDeviceId, aCalculate) ==
899
UINT32_MAX) {
900
rejects.AppendElement(candidateSet[i]);
901
candidateSet.RemoveElementAt(i);
902
} else {
903
++i;
904
}
905
}
906
if (!candidateSet.Length()) {
907
candidateSet.AppendElements(std::move(rejects));
908
}
909
}
910
MOZ_ASSERT(
911
candidateSet.Length(),
912
"advanced constraints filtering step can't reduce candidates to zero");
913
914
// Remaining algorithm is up to the UA.
915
916
TrimLessFitCandidates(candidateSet);
917
918
// Any remaining multiples all have the same distance. A common case of this
919
// occurs when no ideal is specified. Lean toward defaults.
920
uint32_t sameDistance = candidateSet[0].mDistance;
921
{
922
MediaTrackConstraintSet prefs;
923
prefs.mWidth.Construct().SetAsLong() = aPrefs.GetWidth();
924
prefs.mHeight.Construct().SetAsLong() = aPrefs.GetHeight();
925
prefs.mFrameRate.Construct().SetAsDouble() = aPrefs.mFPS;
926
NormalizedConstraintSet normPrefs(prefs, false);
927
928
for (auto& candidate : candidateSet) {
929
candidate.mDistance =
930
GetDistance(candidate.mCapability, normPrefs, aDeviceId, aCalculate);
931
}
932
TrimLessFitCandidates(candidateSet);
933
}
934
935
aCapability = candidateSet[0].mCapability;
936
937
LogCapability("Chosen capability", aCapability, sameDistance);
938
return true;
939
}
940
941
void MediaEngineRemoteVideoSource::GetSettings(
942
MediaTrackSettings& aOutSettings) const {
943
aOutSettings = *mSettings;
944
}
945
946
void MediaEngineRemoteVideoSource::Refresh(int aIndex) {
947
LOG(__PRETTY_FUNCTION__);
948
AssertIsOnOwningThread();
949
950
// NOTE: mCaptureIndex might have changed when allocated!
951
// Use aIndex to update information, but don't change mCaptureIndex!!
952
// Caller looked up this source by uniqueId, so it shouldn't change
953
char deviceName[kMaxDeviceNameLength];
954
char uniqueId[kMaxUniqueIdLength];
955
956
if (camera::GetChildAndCall(&camera::CamerasChild::GetCaptureDevice,
957
mCapEngine, aIndex, deviceName,
958
sizeof(deviceName), uniqueId, sizeof(uniqueId),
959
nullptr)) {
960
return;
961
}
962
963
SetName(NS_ConvertUTF8toUTF16(deviceName));
964
MOZ_ASSERT(mUniqueId.Equals(uniqueId));
965
SetGroupId(NS_ConvertUTF8toUTF16(deviceName));
966
}
967
968
} // namespace mozilla