Source code

Revision control

Other Tools

1
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2
* This Source Code Form is subject to the terms of the Mozilla Public
3
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
4
* You can obtain one at http://mozilla.org/MPL/2.0/. */
5
6
#include "MediaEngineRemoteVideoSource.h"
7
8
#include "CamerasChild.h"
9
#include "MediaManager.h"
10
#include "MediaTrackConstraints.h"
11
#include "mozilla/ErrorNames.h"
12
#include "mozilla/RefPtr.h"
13
#include "nsIPrefService.h"
14
#include "Tracing.h"
15
#include "VideoFrameUtils.h"
16
#include "VideoUtils.h"
17
#include "webrtc/common_video/include/video_frame_buffer.h"
18
#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
19
20
namespace mozilla {
21
22
extern LazyLogModule gMediaManagerLog;
23
#define LOG(...) MOZ_LOG(gMediaManagerLog, LogLevel::Debug, (__VA_ARGS__))
24
#define LOG_FRAME(...) \
25
MOZ_LOG(gMediaManagerLog, LogLevel::Verbose, (__VA_ARGS__))
26
27
using dom::ConstrainLongRange;
28
using dom::MediaSourceEnum;
29
using dom::MediaTrackConstraints;
30
using dom::MediaTrackConstraintSet;
31
using dom::MediaTrackSettings;
32
using dom::VideoFacingModeEnum;
33
34
static Maybe<VideoFacingModeEnum> GetFacingMode(const nsString& aDeviceName) {
35
// Set facing mode based on device name.
36
#if defined(ANDROID)
37
// Names are generated. Example: "Camera 0, Facing back, Orientation 90"
38
//
39
// See media/webrtc/trunk/webrtc/modules/video_capture/android/java/src/org/
40
// webrtc/videoengine/VideoCaptureDeviceInfoAndroid.java
41
42
if (aDeviceName.Find(NS_LITERAL_STRING("Facing back")) != kNotFound) {
43
return Some(VideoFacingModeEnum::Environment);
44
}
45
if (aDeviceName.Find(NS_LITERAL_STRING("Facing front")) != kNotFound) {
46
return Some(VideoFacingModeEnum::User);
47
}
48
#endif // ANDROID
49
#ifdef XP_MACOSX
50
// Kludge to test user-facing cameras on OSX.
51
if (aDeviceName.Find(NS_LITERAL_STRING("Face")) != -1) {
52
return Some(VideoFacingModeEnum::User);
53
}
54
#endif
55
#ifdef XP_WIN
56
// The cameras' name of Surface book are "Microsoft Camera Front" and
57
// "Microsoft Camera Rear" respectively.
58
59
if (aDeviceName.Find(NS_LITERAL_STRING("Front")) != kNotFound) {
60
return Some(VideoFacingModeEnum::User);
61
}
62
if (aDeviceName.Find(NS_LITERAL_STRING("Rear")) != kNotFound) {
63
return Some(VideoFacingModeEnum::Environment);
64
}
65
#endif // WINDOWS
66
67
return Nothing();
68
}
69
70
MediaEngineRemoteVideoSource::MediaEngineRemoteVideoSource(
71
int aIndex, camera::CaptureEngine aCapEngine, bool aScary)
72
: mCaptureIndex(aIndex),
73
mCapEngine(aCapEngine),
74
mScary(aScary),
75
mMutex("MediaEngineRemoteVideoSource::mMutex"),
76
mRescalingBufferPool(/* zero_initialize */ false,
77
/* max_number_of_buffers */ 1),
78
mSettingsUpdatedByFrame(MakeAndAddRef<media::Refcountable<AtomicBool>>()),
79
mSettings(MakeAndAddRef<media::Refcountable<MediaTrackSettings>>()),
80
mFirstFramePromise(mFirstFramePromiseHolder.Ensure(__func__)) {
81
mSettings->mWidth.Construct(0);
82
mSettings->mHeight.Construct(0);
83
mSettings->mFrameRate.Construct(0);
84
Init();
85
}
86
87
MediaEngineRemoteVideoSource::~MediaEngineRemoteVideoSource() {
88
mFirstFramePromiseHolder.RejectIfExists(NS_ERROR_ABORT, __func__);
89
}
90
91
dom::MediaSourceEnum MediaEngineRemoteVideoSource::GetMediaSource() const {
92
switch (mCapEngine) {
93
case camera::BrowserEngine:
94
return MediaSourceEnum::Browser;
95
case camera::CameraEngine:
96
return MediaSourceEnum::Camera;
97
case camera::ScreenEngine:
98
return MediaSourceEnum::Screen;
99
case camera::WinEngine:
100
return MediaSourceEnum::Window;
101
default:
102
MOZ_CRASH();
103
}
104
}
105
106
void MediaEngineRemoteVideoSource::Init() {
107
LOG("%s", __PRETTY_FUNCTION__);
108
AssertIsOnOwningThread();
109
110
char deviceName[kMaxDeviceNameLength];
111
char uniqueId[kMaxUniqueIdLength];
112
if (camera::GetChildAndCall(&camera::CamerasChild::GetCaptureDevice,
113
mCapEngine, mCaptureIndex, deviceName,
114
kMaxDeviceNameLength, uniqueId,
115
kMaxUniqueIdLength, nullptr)) {
116
LOG("Error initializing RemoteVideoSource (GetCaptureDevice)");
117
return;
118
}
119
120
SetName(NS_ConvertUTF8toUTF16(deviceName));
121
mUniqueId = uniqueId;
122
123
mInitDone = true;
124
}
125
126
void MediaEngineRemoteVideoSource::Shutdown() {
127
LOG("%s", __PRETTY_FUNCTION__);
128
AssertIsOnOwningThread();
129
130
if (!mInitDone) {
131
// Already shut down
132
return;
133
}
134
135
if (mState == kStarted) {
136
Stop();
137
}
138
if (mState == kAllocated || mState == kStopped) {
139
Deallocate();
140
}
141
MOZ_ASSERT(mState == kReleased);
142
143
mInitDone = false;
144
}
145
146
void MediaEngineRemoteVideoSource::SetName(nsString aName) {
147
LOG("%s", __PRETTY_FUNCTION__);
148
AssertIsOnOwningThread();
149
150
mDeviceName = std::move(aName);
151
152
Maybe<VideoFacingModeEnum> facingMode;
153
if (GetMediaSource() == MediaSourceEnum::Camera) {
154
// Only cameras can have a facing mode.
155
facingMode = GetFacingMode(mDeviceName);
156
}
157
158
mFacingMode = facingMode.map([](const auto& aFM) {
159
return NS_ConvertUTF8toUTF16(
160
dom::VideoFacingModeEnumValues::strings[uint32_t(aFM)].value);
161
});
162
NS_DispatchToMainThread(NS_NewRunnableFunction(
163
"MediaEngineRemoteVideoSource::SetName (facingMode updater)",
164
[settings = mSettings, mode = mFacingMode]() {
165
if (mode.isNothing()) {
166
settings->mFacingMode.Reset();
167
return;
168
}
169
settings->mFacingMode.Construct(*mode);
170
}));
171
}
172
173
nsString MediaEngineRemoteVideoSource::GetName() const {
174
AssertIsOnOwningThread();
175
176
return mDeviceName;
177
}
178
179
nsCString MediaEngineRemoteVideoSource::GetUUID() const {
180
AssertIsOnOwningThread();
181
182
return mUniqueId;
183
}
184
185
nsString MediaEngineRemoteVideoSource::GetGroupId() const {
186
AssertIsOnOwningThread();
187
188
// The remote video backend doesn't implement group id. We return the device
189
// name and higher layers will correlate this with the name of audio devices.
190
return mDeviceName;
191
}
192
193
nsresult MediaEngineRemoteVideoSource::Allocate(
194
const MediaTrackConstraints& aConstraints, const MediaEnginePrefs& aPrefs,
195
const nsString& aDeviceId, const nsString& aGroupId,
196
const mozilla::ipc::PrincipalInfo& aPrincipalInfo,
197
const char** aOutBadConstraint) {
198
LOG("%s", __PRETTY_FUNCTION__);
199
AssertIsOnOwningThread();
200
201
MOZ_ASSERT(mState == kReleased);
202
203
if (!mInitDone) {
204
LOG("Init not done");
205
return NS_ERROR_FAILURE;
206
}
207
208
NormalizedConstraints constraints(aConstraints);
209
webrtc::CaptureCapability newCapability;
210
LOG("ChooseCapability(kFitness) for mCapability (Allocate) ++");
211
if (!ChooseCapability(constraints, aPrefs, aDeviceId, aGroupId, newCapability,
212
kFitness)) {
213
*aOutBadConstraint = MediaConstraintsHelper::FindBadConstraint(
214
constraints, this, aDeviceId, aGroupId);
215
return NS_ERROR_FAILURE;
216
}
217
LOG("ChooseCapability(kFitness) for mCapability (Allocate) --");
218
219
if (camera::GetChildAndCall(&camera::CamerasChild::AllocateCaptureDevice,
220
mCapEngine, mUniqueId.get(), kMaxUniqueIdLength,
221
mCaptureIndex, aPrincipalInfo)) {
222
return NS_ERROR_FAILURE;
223
}
224
225
{
226
MutexAutoLock lock(mMutex);
227
mState = kAllocated;
228
mCapability = newCapability;
229
}
230
231
LOG("Video device %d allocated", mCaptureIndex);
232
return NS_OK;
233
}
234
235
nsresult MediaEngineRemoteVideoSource::Deallocate() {
236
LOG("%s", __PRETTY_FUNCTION__);
237
AssertIsOnOwningThread();
238
239
MOZ_ASSERT(mState == kStopped || mState == kAllocated);
240
241
if (mStream && IsTrackIDExplicit(mTrackID)) {
242
mStream->EndTrack(mTrackID);
243
}
244
245
{
246
MutexAutoLock lock(mMutex);
247
248
mStream = nullptr;
249
mTrackID = TRACK_NONE;
250
mPrincipal = PRINCIPAL_HANDLE_NONE;
251
mState = kReleased;
252
}
253
254
// Stop() has stopped capture synchronously on the media thread before we get
255
// here, so there are no longer any callbacks on an IPC thread accessing
256
// mImageContainer or mRescalingBufferPool.
257
mImageContainer = nullptr;
258
mRescalingBufferPool.Release();
259
260
LOG("Video device %d deallocated", mCaptureIndex);
261
262
if (camera::GetChildAndCall(&camera::CamerasChild::ReleaseCaptureDevice,
263
mCapEngine, mCaptureIndex)) {
264
MOZ_ASSERT_UNREACHABLE("Couldn't release allocated device");
265
}
266
return NS_OK;
267
}
268
269
void MediaEngineRemoteVideoSource::SetTrack(
270
const RefPtr<SourceMediaStream>& aStream, TrackID aTrackID,
271
const PrincipalHandle& aPrincipal) {
272
LOG("%s", __PRETTY_FUNCTION__);
273
AssertIsOnOwningThread();
274
275
MOZ_ASSERT(mState == kAllocated);
276
MOZ_ASSERT(!mStream);
277
MOZ_ASSERT(mTrackID == TRACK_NONE);
278
MOZ_ASSERT(aStream);
279
MOZ_ASSERT(IsTrackIDExplicit(aTrackID));
280
281
if (!mImageContainer) {
282
mImageContainer = layers::LayerManager::CreateImageContainer(
283
layers::ImageContainer::ASYNCHRONOUS);
284
}
285
286
{
287
MutexAutoLock lock(mMutex);
288
mStream = aStream;
289
mTrackID = aTrackID;
290
mPrincipal = aPrincipal;
291
}
292
aStream->AddTrack(aTrackID, new VideoSegment());
293
}
294
295
nsresult MediaEngineRemoteVideoSource::Start() {
296
LOG("%s", __PRETTY_FUNCTION__);
297
AssertIsOnOwningThread();
298
299
MOZ_ASSERT(mState == kAllocated || mState == kStopped);
300
MOZ_ASSERT(mInitDone);
301
MOZ_ASSERT(mStream);
302
MOZ_ASSERT(IsTrackIDExplicit(mTrackID));
303
304
{
305
MutexAutoLock lock(mMutex);
306
mState = kStarted;
307
}
308
309
mSettingsUpdatedByFrame->mValue = false;
310
311
if (camera::GetChildAndCall(&camera::CamerasChild::StartCapture, mCapEngine,
312
mCaptureIndex, mCapability, this)) {
313
LOG("StartCapture failed");
314
MutexAutoLock lock(mMutex);
315
mState = kStopped;
316
return NS_ERROR_FAILURE;
317
}
318
319
NS_DispatchToMainThread(NS_NewRunnableFunction(
320
"MediaEngineRemoteVideoSource::SetLastCapability",
321
[settings = mSettings, updated = mSettingsUpdatedByFrame,
322
capEngine = mCapEngine, cap = mCapability]() mutable {
323
switch (capEngine) {
324
case camera::ScreenEngine:
325
case camera::WinEngine:
326
// Undo the hack where ideal and max constraints are crammed
327
// together in mCapability for consumption by low-level code. We
328
// don't actually know the real resolution yet, so report min(ideal,
329
// max) for now.
330
// TODO: This can be removed in bug 1453269.
331
cap.width = std::min(cap.width >> 16, cap.width & 0xffff);
332
cap.height = std::min(cap.height >> 16, cap.height & 0xffff);
333
break;
334
default:
335
break;
336
}
337
338
if (!updated->mValue) {
339
settings->mWidth.Value() = cap.width;
340
settings->mHeight.Value() = cap.height;
341
}
342
settings->mFrameRate.Value() = cap.maxFPS;
343
}));
344
345
return NS_OK;
346
}
347
348
nsresult MediaEngineRemoteVideoSource::FocusOnSelectedSource() {
349
LOG("%s", __PRETTY_FUNCTION__);
350
AssertIsOnOwningThread();
351
352
int result;
353
result = camera::GetChildAndCall(&camera::CamerasChild::FocusOnSelectedSource,
354
mCapEngine, mCaptureIndex);
355
return result == 0 ? NS_OK : NS_ERROR_FAILURE;
356
}
357
358
nsresult MediaEngineRemoteVideoSource::Stop() {
359
LOG("%s", __PRETTY_FUNCTION__);
360
AssertIsOnOwningThread();
361
362
if (mState == kStopped || mState == kAllocated) {
363
return NS_OK;
364
}
365
366
MOZ_ASSERT(mState == kStarted);
367
368
if (camera::GetChildAndCall(&camera::CamerasChild::StopCapture, mCapEngine,
369
mCaptureIndex)) {
370
MOZ_DIAGNOSTIC_ASSERT(false, "Stopping a started capture failed");
371
return NS_ERROR_FAILURE;
372
}
373
374
{
375
MutexAutoLock lock(mMutex);
376
mState = kStopped;
377
}
378
379
return NS_OK;
380
}
381
382
nsresult MediaEngineRemoteVideoSource::Reconfigure(
383
const MediaTrackConstraints& aConstraints, const MediaEnginePrefs& aPrefs,
384
const nsString& aDeviceId, const nsString& aGroupId,
385
const char** aOutBadConstraint) {
386
LOG("%s", __PRETTY_FUNCTION__);
387
AssertIsOnOwningThread();
388
389
MOZ_ASSERT(mInitDone);
390
391
NormalizedConstraints constraints(aConstraints);
392
webrtc::CaptureCapability newCapability;
393
LOG("ChooseCapability(kFitness) for mTargetCapability (Reconfigure) ++");
394
if (!ChooseCapability(constraints, aPrefs, aDeviceId, aGroupId, newCapability,
395
kFitness)) {
396
*aOutBadConstraint = MediaConstraintsHelper::FindBadConstraint(
397
constraints, this, aDeviceId, aGroupId);
398
return NS_ERROR_INVALID_ARG;
399
}
400
LOG("ChooseCapability(kFitness) for mTargetCapability (Reconfigure) --");
401
402
if (mCapability == newCapability) {
403
return NS_OK;
404
}
405
406
bool started = mState == kStarted;
407
if (started) {
408
nsresult rv = Stop();
409
if (NS_WARN_IF(NS_FAILED(rv))) {
410
nsAutoCString name;
411
GetErrorName(rv, name);
412
LOG("Video source %p for video device %d Reconfigure() failed "
413
"unexpectedly in Stop(). rv=%s",
414
this, mCaptureIndex, name.Data());
415
return NS_ERROR_UNEXPECTED;
416
}
417
}
418
419
{
420
MutexAutoLock lock(mMutex);
421
// Start() applies mCapability on the device.
422
mCapability = newCapability;
423
}
424
425
if (started) {
426
nsresult rv = Start();
427
if (NS_WARN_IF(NS_FAILED(rv))) {
428
nsAutoCString name;
429
GetErrorName(rv, name);
430
LOG("Video source %p for video device %d Reconfigure() failed "
431
"unexpectedly in Start(). rv=%s",
432
this, mCaptureIndex, name.Data());
433
return NS_ERROR_UNEXPECTED;
434
}
435
}
436
437
return NS_OK;
438
}
439
440
size_t MediaEngineRemoteVideoSource::NumCapabilities() const {
441
AssertIsOnOwningThread();
442
443
mHardcodedCapabilities.Clear();
444
int num = camera::GetChildAndCall(&camera::CamerasChild::NumberOfCapabilities,
445
mCapEngine, mUniqueId.get());
446
447
if (num >= 1) {
448
return num;
449
}
450
451
// The default for devices that don't return discrete capabilities: treat
452
// them as supporting all capabilities orthogonally. E.g. screensharing.
453
// CaptureCapability defaults key values to 0, which means accept any value.
454
mHardcodedCapabilities.AppendElement(webrtc::CaptureCapability());
455
return mHardcodedCapabilities.Length(); // 1
456
}
457
458
webrtc::CaptureCapability MediaEngineRemoteVideoSource::GetCapability(
459
size_t aIndex) const {
460
AssertIsOnOwningThread();
461
webrtc::CaptureCapability result;
462
if (!mHardcodedCapabilities.IsEmpty()) {
463
MOZ_ASSERT(aIndex < mHardcodedCapabilities.Length());
464
result = mHardcodedCapabilities.SafeElementAt(aIndex,
465
webrtc::CaptureCapability());
466
}
467
camera::GetChildAndCall(&camera::CamerasChild::GetCaptureCapability,
468
mCapEngine, mUniqueId.get(), aIndex, result);
469
return result;
470
}
471
472
int MediaEngineRemoteVideoSource::DeliverFrame(
473
uint8_t* aBuffer, const camera::VideoFrameProperties& aProps) {
474
// Cameras IPC thread - take great care with accessing members!
475
476
int32_t req_max_width;
477
int32_t req_max_height;
478
int32_t req_ideal_width;
479
int32_t req_ideal_height;
480
{
481
MutexAutoLock lock(mMutex);
482
MOZ_ASSERT(mState == kStarted);
483
// TODO: These can be removed in bug 1453269.
484
req_max_width = mCapability.width & 0xffff;
485
req_max_height = mCapability.height & 0xffff;
486
req_ideal_width = (mCapability.width >> 16) & 0xffff;
487
req_ideal_height = (mCapability.height >> 16) & 0xffff;
488
}
489
490
// This is only used in the case of screen sharing, see bug 1453269.
491
const int32_t target_width = aProps.width();
492
const int32_t target_height = aProps.height();
493
494
if (aProps.rotation() == 90 || aProps.rotation() == 270) {
495
// This frame is rotated, so what was negotiated as width is now height,
496
// and vice versa.
497
std::swap(req_max_width, req_max_height);
498
std::swap(req_ideal_width, req_ideal_height);
499
}
500
501
int32_t dst_max_width = req_max_width == 0
502
? aProps.width()
503
: std::min(req_max_width, aProps.width());
504
int32_t dst_max_height = req_max_height == 0
505
? aProps.height()
506
: std::min(req_max_height, aProps.height());
507
// This logic works for both camera and screen sharing case.
508
// for camera case, req_ideal_width and req_ideal_height is 0.
509
// The following snippet will set dst_width to dst_max_width and dst_height to
510
// dst_max_height
511
int32_t dst_width = std::min(
512
req_ideal_width > 0 ? req_ideal_width : aProps.width(), dst_max_width);
513
int32_t dst_height =
514
std::min(req_ideal_height > 0 ? req_ideal_height : aProps.height(),
515
dst_max_height);
516
517
// Apply scaling for screen sharing, see bug 1453269.
518
switch (mCapEngine) {
519
case camera::ScreenEngine:
520
case camera::WinEngine: {
521
// scale to average of portrait and landscape
522
float scale_width = (float)dst_width / (float)aProps.width();
523
float scale_height = (float)dst_height / (float)aProps.height();
524
float scale = (scale_width + scale_height) / 2;
525
dst_width = (int)(scale * target_width);
526
dst_height = (int)(scale * target_height);
527
528
// if scaled rectangle exceeds max rectangle, scale to minimum of portrait
529
// and landscape
530
if (dst_width > dst_max_width || dst_height > dst_max_height) {
531
scale_width = (float)dst_max_width / (float)dst_width;
532
scale_height = (float)dst_max_height / (float)dst_height;
533
scale = std::min(scale_width, scale_height);
534
dst_width = (int32_t)(scale * dst_width);
535
dst_height = (int32_t)(scale * dst_height);
536
}
537
break;
538
}
539
default: {
540
break;
541
}
542
}
543
544
rtc::Callback0<void> callback_unused;
545
rtc::scoped_refptr<webrtc::I420BufferInterface> buffer =
546
new rtc::RefCountedObject<webrtc::WrappedI420Buffer>(
547
aProps.width(), aProps.height(), aBuffer, aProps.yStride(),
548
aBuffer + aProps.yAllocatedSize(), aProps.uStride(),
549
aBuffer + aProps.yAllocatedSize() + aProps.uAllocatedSize(),
550
aProps.vStride(), callback_unused);
551
552
if ((dst_width != aProps.width() || dst_height != aProps.height()) &&
553
dst_width <= aProps.width() && dst_height <= aProps.height()) {
554
// Destination resolution is smaller than source buffer. We'll rescale.
555
rtc::scoped_refptr<webrtc::I420Buffer> scaledBuffer =
556
mRescalingBufferPool.CreateBuffer(dst_width, dst_height);
557
if (!scaledBuffer) {
558
MOZ_ASSERT_UNREACHABLE(
559
"We might fail to allocate a buffer, but with this "
560
"being a recycling pool that shouldn't happen");
561
return 0;
562
}
563
scaledBuffer->CropAndScaleFrom(*buffer);
564
buffer = scaledBuffer;
565
}
566
567
layers::PlanarYCbCrData data;
568
data.mYChannel = const_cast<uint8_t*>(buffer->DataY());
569
data.mYSize = IntSize(buffer->width(), buffer->height());
570
data.mYStride = buffer->StrideY();
571
MOZ_ASSERT(buffer->StrideU() == buffer->StrideV());
572
data.mCbCrStride = buffer->StrideU();
573
data.mCbChannel = const_cast<uint8_t*>(buffer->DataU());
574
data.mCrChannel = const_cast<uint8_t*>(buffer->DataV());
575
data.mCbCrSize =
576
IntSize((buffer->width() + 1) / 2, (buffer->height() + 1) / 2);
577
data.mPicX = 0;
578
data.mPicY = 0;
579
data.mPicSize = IntSize(buffer->width(), buffer->height());
580
data.mYUVColorSpace = gfx::YUVColorSpace::BT601;
581
582
RefPtr<layers::PlanarYCbCrImage> image =
583
mImageContainer->CreatePlanarYCbCrImage();
584
if (!image->CopyData(data)) {
585
MOZ_ASSERT_UNREACHABLE(
586
"We might fail to allocate a buffer, but with this "
587
"being a recycling container that shouldn't happen");
588
return 0;
589
}
590
591
#ifdef DEBUG
592
static uint32_t frame_num = 0;
593
LOG_FRAME(
594
"frame %d (%dx%d)->(%dx%d); rotation %d, timeStamp %u, ntpTimeMs %" PRIu64
595
", renderTimeMs %" PRIu64,
596
frame_num++, aProps.width(), aProps.height(), dst_width, dst_height,
597
aProps.rotation(), aProps.timeStamp(), aProps.ntpTimeMs(),
598
aProps.renderTimeMs());
599
#endif
600
601
if (mImageSize.width != dst_width || mImageSize.height != dst_height) {
602
NS_DispatchToMainThread(NS_NewRunnableFunction(
603
"MediaEngineRemoteVideoSource::FrameSizeChange",
604
[settings = mSettings, updated = mSettingsUpdatedByFrame,
605
holder = std::move(mFirstFramePromiseHolder), dst_width,
606
dst_height]() mutable {
607
settings->mWidth.Value() = dst_width;
608
settings->mHeight.Value() = dst_height;
609
updated->mValue = true;
610
// Since mImageSize was initialized to (0,0), we end up here on the
611
// arrival of the first frame. We resolve the promise representing
612
// arrival of first frame, after correct settings values have been
613
// made available (Resolve() is idempotent if already resolved).
614
holder.ResolveIfExists(true, __func__);
615
}));
616
}
617
618
{
619
MutexAutoLock lock(mMutex);
620
MOZ_ASSERT(mState == kStarted);
621
VideoSegment segment;
622
mImageSize = image->GetSize();
623
segment.AppendFrame(image.forget(), mImageSize, mPrincipal);
624
mStream->AppendToTrack(mTrackID, &segment);
625
}
626
627
return 0;
628
}
629
630
uint32_t MediaEngineRemoteVideoSource::GetDistance(
631
const webrtc::CaptureCapability& aCandidate,
632
const NormalizedConstraintSet& aConstraints, const nsString& aDeviceId,
633
const nsString& aGroupId, const DistanceCalculation aCalculate) const {
634
if (aCalculate == kFeasibility) {
635
return GetFeasibilityDistance(aCandidate, aConstraints, aDeviceId,
636
aGroupId);
637
}
638
return GetFitnessDistance(aCandidate, aConstraints, aDeviceId, aGroupId);
639
}
640
641
uint32_t MediaEngineRemoteVideoSource::GetFitnessDistance(
642
const webrtc::CaptureCapability& aCandidate,
643
const NormalizedConstraintSet& aConstraints, const nsString& aDeviceId,
644
const nsString& aGroupId) const {
645
AssertIsOnOwningThread();
646
647
// Treat width|height|frameRate == 0 on capability as "can do any".
648
// This allows for orthogonal capabilities that are not in discrete steps.
649
650
typedef MediaConstraintsHelper H;
651
uint64_t distance =
652
uint64_t(H::FitnessDistance(Some(aDeviceId), aConstraints.mDeviceId)) +
653
uint64_t(H::FitnessDistance(Some(aGroupId), aConstraints.mGroupId)) +
654
uint64_t(H::FitnessDistance(mFacingMode, aConstraints.mFacingMode)) +
655
uint64_t(aCandidate.width ? H::FitnessDistance(int32_t(aCandidate.width),
656
aConstraints.mWidth)
657
: 0) +
658
uint64_t(aCandidate.height
659
? H::FitnessDistance(int32_t(aCandidate.height),
660
aConstraints.mHeight)
661
: 0) +
662
uint64_t(aCandidate.maxFPS ? H::FitnessDistance(double(aCandidate.maxFPS),
663
aConstraints.mFrameRate)
664
: 0);
665
return uint32_t(std::min(distance, uint64_t(UINT32_MAX)));
666
}
667
668
uint32_t MediaEngineRemoteVideoSource::GetFeasibilityDistance(
669
const webrtc::CaptureCapability& aCandidate,
670
const NormalizedConstraintSet& aConstraints, const nsString& aDeviceId,
671
const nsString& aGroupId) const {
672
AssertIsOnOwningThread();
673
674
// Treat width|height|frameRate == 0 on capability as "can do any".
675
// This allows for orthogonal capabilities that are not in discrete steps.
676
677
typedef MediaConstraintsHelper H;
678
uint64_t distance =
679
uint64_t(H::FitnessDistance(Some(aDeviceId), aConstraints.mDeviceId)) +
680
uint64_t(H::FitnessDistance(Some(aGroupId), aConstraints.mGroupId)) +
681
uint64_t(H::FitnessDistance(mFacingMode, aConstraints.mFacingMode)) +
682
uint64_t(aCandidate.width
683
? H::FeasibilityDistance(int32_t(aCandidate.width),
684
aConstraints.mWidth)
685
: 0) +
686
uint64_t(aCandidate.height
687
? H::FeasibilityDistance(int32_t(aCandidate.height),
688
aConstraints.mHeight)
689
: 0) +
690
uint64_t(aCandidate.maxFPS
691
? H::FeasibilityDistance(double(aCandidate.maxFPS),
692
aConstraints.mFrameRate)
693
: 0);
694
return uint32_t(std::min(distance, uint64_t(UINT32_MAX)));
695
}
696
697
// Find best capability by removing inferiors. May leave >1 of equal distance
698
699
/* static */
700
void MediaEngineRemoteVideoSource::TrimLessFitCandidates(
701
nsTArray<CapabilityCandidate>& aSet) {
702
uint32_t best = UINT32_MAX;
703
for (auto& candidate : aSet) {
704
if (best > candidate.mDistance) {
705
best = candidate.mDistance;
706
}
707
}
708
for (size_t i = 0; i < aSet.Length();) {
709
if (aSet[i].mDistance > best) {
710
aSet.RemoveElementAt(i);
711
} else {
712
++i;
713
}
714
}
715
MOZ_ASSERT(aSet.Length());
716
}
717
718
uint32_t MediaEngineRemoteVideoSource::GetBestFitnessDistance(
719
const nsTArray<const NormalizedConstraintSet*>& aConstraintSets,
720
const nsString& aDeviceId, const nsString& aGroupId) const {
721
AssertIsOnOwningThread();
722
723
size_t num = NumCapabilities();
724
nsTArray<CapabilityCandidate> candidateSet;
725
for (size_t i = 0; i < num; i++) {
726
candidateSet.AppendElement(CapabilityCandidate(GetCapability(i)));
727
}
728
729
bool first = true;
730
for (const NormalizedConstraintSet* ns : aConstraintSets) {
731
for (size_t i = 0; i < candidateSet.Length();) {
732
auto& candidate = candidateSet[i];
733
uint32_t distance =
734
GetFitnessDistance(candidate.mCapability, *ns, aDeviceId, aGroupId);
735
if (distance == UINT32_MAX) {
736
candidateSet.RemoveElementAt(i);
737
} else {
738
++i;
739
if (first) {
740
candidate.mDistance = distance;
741
}
742
}
743
}
744
first = false;
745
}
746
if (!candidateSet.Length()) {
747
return UINT32_MAX;
748
}
749
TrimLessFitCandidates(candidateSet);
750
return candidateSet[0].mDistance;
751
}
752
753
static void LogCapability(const char* aHeader,
754
const webrtc::CaptureCapability& aCapability,
755
uint32_t aDistance) {
756
static const char* const codec[] = {"VP8", "VP9", "H264",
757
"I420", "RED", "ULPFEC",
758
"Generic codec", "Unknown codec"};
759
760
LOG("%s: %4u x %4u x %2u maxFps, %s. Distance = %" PRIu32, aHeader,
761
aCapability.width, aCapability.height, aCapability.maxFPS,
762
codec[std::min(std::max(uint32_t(0), uint32_t(aCapability.videoType)),
763
uint32_t(sizeof(codec) / sizeof(*codec) - 1))],
764
aDistance);
765
}
766
767
bool MediaEngineRemoteVideoSource::ChooseCapability(
768
const NormalizedConstraints& aConstraints, const MediaEnginePrefs& aPrefs,
769
const nsString& aDeviceId, const nsString& aGroupId,
770
webrtc::CaptureCapability& aCapability,
771
const DistanceCalculation aCalculate) {
772
LOG("%s", __PRETTY_FUNCTION__);
773
AssertIsOnOwningThread();
774
775
if (MOZ_LOG_TEST(gMediaManagerLog, LogLevel::Debug)) {
776
LOG("ChooseCapability: prefs: %dx%d @%dfps", aPrefs.GetWidth(),
777
aPrefs.GetHeight(), aPrefs.mFPS);
778
MediaConstraintsHelper::LogConstraints(aConstraints);
779
if (!aConstraints.mAdvanced.empty()) {
780
LOG("Advanced array[%zu]:", aConstraints.mAdvanced.size());
781
for (auto& advanced : aConstraints.mAdvanced) {
782
MediaConstraintsHelper::LogConstraints(advanced);
783
}
784
}
785
}
786
787
switch (mCapEngine) {
788
case camera::ScreenEngine:
789
case camera::WinEngine: {
790
FlattenedConstraints c(aConstraints);
791
// The actual resolution to constrain around is not easy to find ahead of
792
// time (and may in fact change over time), so as a hack, we push ideal
793
// and max constraints down to desktop_capture_impl.cc and finish the
794
// algorithm there.
795
// TODO: This can be removed in bug 1453269.
796
aCapability.width =
797
(std::min(0xffff, c.mWidth.mIdeal.valueOr(0)) & 0xffff) << 16 |
798
(std::min(0xffff, c.mWidth.mMax) & 0xffff);
799
aCapability.height =
800
(std::min(0xffff, c.mHeight.mIdeal.valueOr(0)) & 0xffff) << 16 |
801
(std::min(0xffff, c.mHeight.mMax) & 0xffff);
802
aCapability.maxFPS =
803
c.mFrameRate.Clamp(c.mFrameRate.mIdeal.valueOr(aPrefs.mFPS));
804
return true;
805
}
806
default:
807
break;
808
}
809
810
nsTArray<CapabilityCandidate> candidateSet;
811
size_t num = NumCapabilities();
812
for (size_t i = 0; i < num; i++) {
813
candidateSet.AppendElement(CapabilityCandidate(GetCapability(i)));
814
}
815
816
if (!mHardcodedCapabilities.IsEmpty() && mCapEngine == camera::CameraEngine) {
817
// We have a hardcoded capability, which means this camera didn't report
818
// discrete capabilities. It might still allow a ranged capability, so we
819
// add a couple of default candidates based on prefs and constraints.
820
// The chosen candidate will be propagated to StartCapture() which will fail
821
// for an invalid candidate.
822
MOZ_DIAGNOSTIC_ASSERT(mHardcodedCapabilities.Length() == 1);
823
MOZ_DIAGNOSTIC_ASSERT(candidateSet.Length() == 1);
824
candidateSet.Clear();
825
826
FlattenedConstraints c(aConstraints);
827
// Reuse the code across both the low-definition (`false`) pref and
828
// the high-definition (`true`) pref.
829
// If there are constraints we try to satisfy them but we default to prefs.
830
// Note that since constraints are from content and can literally be
831
// anything we put (rather generous) caps on them.
832
for (bool isHd : {false, true}) {
833
webrtc::CaptureCapability cap;
834
int32_t prefWidth = aPrefs.GetWidth(isHd);
835
int32_t prefHeight = aPrefs.GetHeight(isHd);
836
837
cap.width = c.mWidth.Get(prefWidth);
838
cap.width = std::max(0, std::min(cap.width, 7680));
839
840
cap.height = c.mHeight.Get(prefHeight);
841
cap.height = std::max(0, std::min(cap.height, 4320));
842
843
cap.maxFPS = c.mFrameRate.Get(aPrefs.mFPS);
844
cap.maxFPS = std::max(0, std::min(cap.maxFPS, 480));
845
846
if (cap.width != prefWidth) {
847
// Width was affected by constraints.
848
// We'll adjust the height too so the aspect ratio is retained.
849
cap.height = cap.width * prefHeight / prefWidth;
850
} else if (cap.height != prefHeight) {
851
// Height was affected by constraints but not width.
852
// We'll adjust the width too so the aspect ratio is retained.
853
cap.width = cap.height * prefWidth / prefHeight;
854
}
855
856
if (candidateSet.Contains(cap, CapabilityComparator())) {
857
continue;
858
}
859
LogCapability("Hardcoded capability", cap, 0);
860
candidateSet.AppendElement(cap);
861
}
862
}
863
864
// First, filter capabilities by required constraints (min, max, exact).
865
866
for (size_t i = 0; i < candidateSet.Length();) {
867
auto& candidate = candidateSet[i];
868
candidate.mDistance = GetDistance(candidate.mCapability, aConstraints,
869
aDeviceId, aGroupId, aCalculate);
870
LogCapability("Capability", candidate.mCapability, candidate.mDistance);
871
if (candidate.mDistance == UINT32_MAX) {
872
candidateSet.RemoveElementAt(i);
873
} else {
874
++i;
875
}
876
}
877
878
if (candidateSet.IsEmpty()) {
879
LOG("failed to find capability match from %zu choices",
880
candidateSet.Length());
881
return false;
882
}
883
884
// Filter further with all advanced constraints (that don't overconstrain).
885
886
for (const auto& cs : aConstraints.mAdvanced) {
887
nsTArray<CapabilityCandidate> rejects;
888
for (size_t i = 0; i < candidateSet.Length();) {
889
if (GetDistance(candidateSet[i].mCapability, cs, aDeviceId, aGroupId,
890
aCalculate) == UINT32_MAX) {
891
rejects.AppendElement(candidateSet[i]);
892
candidateSet.RemoveElementAt(i);
893
} else {
894
++i;
895
}
896
}
897
if (!candidateSet.Length()) {
898
candidateSet.AppendElements(std::move(rejects));
899
}
900
}
901
MOZ_ASSERT(
902
candidateSet.Length(),
903
"advanced constraints filtering step can't reduce candidates to zero");
904
905
// Remaining algorithm is up to the UA.
906
907
TrimLessFitCandidates(candidateSet);
908
909
// Any remaining multiples all have the same distance. A common case of this
910
// occurs when no ideal is specified. Lean toward defaults.
911
uint32_t sameDistance = candidateSet[0].mDistance;
912
{
913
MediaTrackConstraintSet prefs;
914
prefs.mWidth.Construct().SetAsLong() = aPrefs.GetWidth();
915
prefs.mHeight.Construct().SetAsLong() = aPrefs.GetHeight();
916
prefs.mFrameRate.Construct().SetAsDouble() = aPrefs.mFPS;
917
NormalizedConstraintSet normPrefs(prefs, false);
918
919
for (auto& candidate : candidateSet) {
920
candidate.mDistance = GetDistance(candidate.mCapability, normPrefs,
921
aDeviceId, aGroupId, aCalculate);
922
}
923
TrimLessFitCandidates(candidateSet);
924
}
925
926
aCapability = candidateSet[0].mCapability;
927
928
LogCapability("Chosen capability", aCapability, sameDistance);
929
return true;
930
}
931
932
void MediaEngineRemoteVideoSource::GetSettings(
933
MediaTrackSettings& aOutSettings) const {
934
aOutSettings = *mSettings;
935
}
936
937
void MediaEngineRemoteVideoSource::Refresh(int aIndex) {
938
LOG("%s", __PRETTY_FUNCTION__);
939
AssertIsOnOwningThread();
940
941
// NOTE: mCaptureIndex might have changed when allocated!
942
// Use aIndex to update information, but don't change mCaptureIndex!!
943
// Caller looked up this source by uniqueId, so it shouldn't change
944
char deviceName[kMaxDeviceNameLength];
945
char uniqueId[kMaxUniqueIdLength];
946
947
if (camera::GetChildAndCall(&camera::CamerasChild::GetCaptureDevice,
948
mCapEngine, aIndex, deviceName,
949
sizeof(deviceName), uniqueId, sizeof(uniqueId),
950
nullptr)) {
951
return;
952
}
953
954
SetName(NS_ConvertUTF8toUTF16(deviceName));
955
MOZ_DIAGNOSTIC_ASSERT(mUniqueId.Equals(uniqueId));
956
}
957
958
} // namespace mozilla