Source code

Revision control

Other Tools

/* -*- Mode: c++; c-basic-offset: 2; indent-tabs-mode: nil; tab-width: 40 -*- */
/* vim: set ts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "MediaManager.h"
#include "AudioCaptureTrack.h"
#include "AudioDeviceInfo.h"
#include "AudioStreamTrack.h"
#include "CubebDeviceEnumerator.h"
#include "MediaTimer.h"
#include "MediaTrackConstraints.h"
#include "MediaTrackGraphImpl.h"
#include "MediaTrackListener.h"
#include "ThreadSafeRefcountingWithMainThreadDestruction.h"
#include "VideoStreamTrack.h"
#include "VideoUtils.h"
#include "mozilla/Base64.h"
#include "mozilla/MozPromise.h"
#include "mozilla/NullPrincipal.h"
#include "mozilla/PeerIdentity.h"
#include "mozilla/PermissionDelegateHandler.h"
#include "mozilla/Sprintf.h"
#include "mozilla/StaticPrefs_media.h"
#include "mozilla/Telemetry.h"
#include "mozilla/Types.h"
#include "mozilla/dom/BindingDeclarations.h"
#include "mozilla/dom/Document.h"
#include "mozilla/dom/Element.h"
#include "mozilla/dom/FeaturePolicyUtils.h"
#include "mozilla/dom/File.h"
#include "mozilla/dom/GetUserMediaRequestBinding.h"
#include "mozilla/dom/MediaDeviceInfo.h"
#include "mozilla/dom/MediaDevices.h"
#include "mozilla/dom/MediaStreamBinding.h"
#include "mozilla/dom/MediaStreamTrackBinding.h"
#include "mozilla/dom/Promise.h"
#include "mozilla/dom/UserActivation.h"
#include "mozilla/dom/WindowContext.h"
#include "mozilla/ipc/BackgroundChild.h"
#include "mozilla/media/MediaChild.h"
#include "mozilla/media/MediaTaskUtils.h"
#include "nsAppDirectoryServiceDefs.h"
#include "nsArray.h"
#include "nsContentUtils.h"
#include "nsGlobalWindow.h"
#include "nsHashPropertyBag.h"
#include "nsICryptoHMAC.h"
#include "nsIDocShell.h"
#include "nsIEventTarget.h"
#include "nsIKeyModule.h"
#include "nsIPermissionManager.h"
#include "nsIUUIDGenerator.h"
#include "nsJSUtils.h"
#include "nsNetCID.h"
#include "nsNetUtil.h"
#include "nsPIDOMWindow.h"
#include "nsProxyRelease.h"
#include "nsVariant.h"
#include "nspr.h"
#include "nss.h"
#include "pk11pub.h"
/* Using WebRTC backend on Desktops (Mac, Windows, Linux), otherwise default */
#include "MediaEngineDefault.h"
#if defined(MOZ_WEBRTC)
# include "MediaEngineWebRTC.h"
# include "browser_logging/WebRtcLog.h"
# include "webrtc/modules/audio_processing/include/audio_processing.h"
#endif
#if defined(XP_WIN)
# include <iphlpapi.h>
# include <objbase.h>
# include <tchar.h>
# include <winsock2.h>
# include "mozilla/WindowsVersion.h"
#endif
// XXX Workaround for bug 986974 to maintain the existing broken semantics
template <>
struct nsIMediaDevice::COMTypeInfo<mozilla::MediaDevice, void> {
static const nsIID kIID;
};
const nsIID nsIMediaDevice::COMTypeInfo<mozilla::MediaDevice, void>::kIID =
NS_IMEDIADEVICE_IID;
// A specialization of nsMainThreadPtrHolder for
// mozilla::dom::CallbackObjectHolder. See documentation for
// nsMainThreadPtrHolder in nsProxyRelease.h. This specialization lets us avoid
// wrapping the CallbackObjectHolder into a separate refcounted object.
template <class WebIDLCallbackT, class XPCOMCallbackT>
class nsMainThreadPtrHolder<
mozilla::dom::CallbackObjectHolder<WebIDLCallbackT, XPCOMCallbackT>>
final {
typedef mozilla::dom::CallbackObjectHolder<WebIDLCallbackT, XPCOMCallbackT>
Holder;
public:
nsMainThreadPtrHolder(const char* aName, Holder&& aHolder)
: mHolder(std::move(aHolder))
#ifndef RELEASE_OR_BETA
,
mName(aName)
#endif
{
MOZ_ASSERT(NS_IsMainThread());
}
private:
// We can be released on any thread.
~nsMainThreadPtrHolder() {
if (NS_IsMainThread()) {
mHolder.Reset();
} else if (mHolder.GetISupports()) {
nsCOMPtr<nsIEventTarget> target = do_GetMainThread();
MOZ_ASSERT(target);
NS_ProxyRelease(
#ifdef RELEASE_OR_BETA
nullptr,
#else
mName,
#endif
target, mHolder.Forget());
}
}
public:
Holder* get() {
// Nobody should be touching the raw pointer off-main-thread.
if (MOZ_UNLIKELY(!NS_IsMainThread())) {
NS_ERROR("Can't dereference nsMainThreadPtrHolder off main thread");
MOZ_CRASH();
}
return &mHolder;
}
bool operator!() const { return !mHolder; }
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(nsMainThreadPtrHolder<Holder>)
private:
// Our holder.
Holder mHolder;
#ifndef RELEASE_OR_BETA
const char* mName = nullptr;
#endif
// Copy constructor and operator= not implemented. Once constructed, the
// holder is immutable.
Holder& operator=(const nsMainThreadPtrHolder& aOther) = delete;
nsMainThreadPtrHolder(const nsMainThreadPtrHolder& aOther) = delete;
};
namespace mozilla {
LazyLogModule gMediaManagerLog("MediaManager");
#define LOG(...) MOZ_LOG(gMediaManagerLog, LogLevel::Debug, (__VA_ARGS__))
class LocalTrackSource;
using dom::CallerType;
using dom::ConstrainDOMStringParameters;
using dom::ConstrainDoubleRange;
using dom::ConstrainLongRange;
using dom::DisplayMediaStreamConstraints;
using dom::Document;
using dom::Element;
using dom::FeaturePolicyUtils;
using dom::File;
using dom::GetUserMediaRequest;
using dom::MediaDeviceKind;
using dom::MediaDevices;
using dom::MediaSourceEnum;
using dom::MediaStreamConstraints;
using dom::MediaStreamError;
using dom::MediaStreamTrack;
using dom::MediaStreamTrackSource;
using dom::MediaTrackConstraints;
using dom::MediaTrackConstraintSet;
using dom::MediaTrackSettings;
using dom::MozGetUserMediaDevicesSuccessCallback;
using dom::OwningBooleanOrMediaTrackConstraints;
using dom::OwningStringOrStringSequence;
using dom::OwningStringOrStringSequenceOrConstrainDOMStringParameters;
using dom::Promise;
using dom::Sequence;
using dom::UserActivation;
using media::NewRunnableFrom;
using media::NewTaskFrom;
using media::Refcountable;
static Atomic<bool> sHasShutdown;
struct DeviceState {
DeviceState(RefPtr<MediaDevice> aDevice,
RefPtr<LocalTrackSource> aTrackSource, bool aOffWhileDisabled)
: mOffWhileDisabled(aOffWhileDisabled),
mDevice(std::move(aDevice)),
mTrackSource(std::move(aTrackSource)) {
MOZ_ASSERT(mDevice);
MOZ_ASSERT(mTrackSource);
}
// true if we have stopped mDevice, this is a terminal state.
// MainThread only.
bool mStopped = false;
// true if mDevice is currently enabled.
// A device must be both enabled and unmuted to be turned on and capturing.
// MainThread only.
bool mDeviceEnabled = false;
// true if mDevice is currently muted.
// A device that is either muted or disabled is turned off and not capturing.
// MainThread only.
bool mDeviceMuted;
// true if the application has currently enabled mDevice.
// MainThread only.
bool mTrackEnabled = false;
// Time when the application last enabled mDevice.
// MainThread only.
TimeStamp mTrackEnabledTime;
// true if an operation to Start() or Stop() mDevice has been dispatched to
// the media thread and is not finished yet.
// MainThread only.
bool mOperationInProgress = false;
// true if we are allowed to turn off the underlying source while all tracks
// are disabled. Only affects disabling; always turns off on user-agent mute.
// MainThread only.
bool mOffWhileDisabled = false;
// Timer triggered by a MediaStreamTrackSource signaling that all tracks got
// disabled. When the timer fires we initiate Stop()ing mDevice.
// If set we allow dynamically stopping and starting mDevice.
// Any thread.
const RefPtr<MediaTimer> mDisableTimer = new MediaTimer();
// The underlying device we keep state for. Always non-null.
// Threadsafe access, but see method declarations for individual constraints.
const RefPtr<MediaDevice> mDevice;
// The MediaStreamTrackSource for any tracks (original and clones) originating
// from this device. Always non-null. Threadsafe access, but see method
// declarations for individual constraints.
const RefPtr<LocalTrackSource> mTrackSource;
};
/**
* This mimics the capture state from nsIMediaManagerService.
*/
enum class CaptureState : uint16_t {
Off = nsIMediaManagerService::STATE_NOCAPTURE,
Enabled = nsIMediaManagerService::STATE_CAPTURE_ENABLED,
Disabled = nsIMediaManagerService::STATE_CAPTURE_DISABLED,
};
static CaptureState CombineCaptureState(CaptureState aFirst,
CaptureState aSecond) {
if (aFirst == CaptureState::Enabled || aSecond == CaptureState::Enabled) {
return CaptureState::Enabled;
}
if (aFirst == CaptureState::Disabled || aSecond == CaptureState::Disabled) {
return CaptureState::Disabled;
}
MOZ_ASSERT(aFirst == CaptureState::Off);
MOZ_ASSERT(aSecond == CaptureState::Off);
return CaptureState::Off;
}
static uint16_t FromCaptureState(CaptureState aState) {
MOZ_ASSERT(aState == CaptureState::Off || aState == CaptureState::Enabled ||
aState == CaptureState::Disabled);
return static_cast<uint16_t>(aState);
}
void MediaManager::CallOnError(GetUserMediaErrorCallback& aCallback,
MediaStreamError& aError) {
aCallback.Call(aError);
}
void MediaManager::CallOnSuccess(GetUserMediaSuccessCallback& aCallback,
DOMMediaStream& aStream) {
aCallback.Call(aStream);
}
/**
* SourceListener has threadsafe refcounting for use across the main, media and
* MTG threads. But it has a non-threadsafe SupportsWeakPtr for WeakPtr usage
* only from main thread, to ensure that garbage- and cycle-collected objects
* don't hold a reference to it during late shutdown.
*/
class SourceListener : public SupportsWeakPtr {
public:
typedef MozPromise<bool /* aIgnored */, RefPtr<MediaMgrError>, true>
SourceListenerPromise;
NS_INLINE_DECL_THREADSAFE_REFCOUNTING_WITH_MAIN_THREAD_DESTRUCTION(
SourceListener)
SourceListener();
/**
* Registers this source listener as belonging to the given window listener.
*/
void Register(GetUserMediaWindowListener* aListener);
/**
* Marks this listener as active and creates internal device states.
*/
void Activate(RefPtr<MediaDevice> aAudioDevice,
RefPtr<LocalTrackSource> aAudioTrackSource,
RefPtr<MediaDevice> aVideoDevice,
RefPtr<LocalTrackSource> aVideoTrackSource,
bool aStartVideoMuted, bool aStartAudioMuted);
/**
* Posts a task to initialize and start all associated devices.
*/
RefPtr<SourceListenerPromise> InitializeAsync();
/**
* Stops all live tracks, ends the associated MediaTrack and cleans up the
* weak reference to the associated window listener.
* This will also tell the window listener to remove its hard reference to
* this SourceListener, so any caller will need to keep its own hard ref.
*/
void Stop();
/**
* Posts a task to stop the device associated with aTrack and notifies the
* associated window listener that a track was stopped.
* Should this track be the last live one to be stopped, we'll also call Stop.
* This might tell the window listener to remove its hard reference to this
* SourceListener, so any caller will need to keep its own hard ref.
*/
void StopTrack(MediaTrack* aTrack);
/**
* Like StopTrack with the audio device's track.
*/
void StopAudioTrack();
/**
* Like StopTrack with the video device's track.
*/
void StopVideoTrack();
/**
* Gets the main thread MediaTrackSettings from the MediaEngineSource
* associated with aTrack.
*/
void GetSettingsFor(MediaTrack* aTrack,
MediaTrackSettings& aOutSettings) const;
/**
* Posts a task to set the enabled state of the device associated with
* aTrack to aEnabled and notifies the associated window listener that a
* track's state has changed.
*
* Turning the hardware off while the device is disabled is supported for:
* - Camera (enabled by default, controlled by pref
* "media.getusermedia.camera.off_while_disabled.enabled")
* - Microphone (disabled by default, controlled by pref
* "media.getusermedia.microphone.off_while_disabled.enabled")
* Screen-, app-, or windowsharing is not supported at this time.
*
* The behavior is also different between disabling and enabling a device.
* While enabling is immediate, disabling only happens after a delay.
* This is now defaulting to 3 seconds but can be overriden by prefs:
* - "media.getusermedia.camera.off_while_disabled.delay_ms" and
* - "media.getusermedia.microphone.off_while_disabled.delay_ms".
*
* The delay is in place to prevent misuse by malicious sites. If a track is
* re-enabled before the delay has passed, the device will not be touched
* until another disable followed by the full delay happens.
*/
void SetEnabledFor(MediaTrack* aTrack, bool aEnabled);
/**
* Posts a task to set the muted state of the device associated with
* aTrackSource to aMuted and notifies the associated window listener that a
* track's state has changed.
*
* Turning the hardware off while the device is muted is supported for:
* - Camera (enabled by default, controlled by pref
* "media.getusermedia.camera.off_while_disabled.enabled")
* - Microphone (disabled by default, controlled by pref
* "media.getusermedia.microphone.off_while_disabled.enabled")
* Screen-, app-, or windowsharing is not supported at this time.
*/
void SetMutedFor(LocalTrackSource* aTrackSource, bool aMuted);
/**
* Stops all screen/app/window/audioCapture sharing, but not camera or
* microphone.
*/
void StopSharing();
/**
* Mutes or unmutes the associated video device if it is a camera.
*/
void MuteOrUnmuteCamera(bool aMute);
void MuteOrUnmuteMicrophone(bool aMute);
MediaDevice* GetAudioDevice() const {
return mAudioDeviceState ? mAudioDeviceState->mDevice.get() : nullptr;
}
MediaDevice* GetVideoDevice() const {
return mVideoDeviceState ? mVideoDeviceState->mDevice.get() : nullptr;
}
bool Activated() const { return mAudioDeviceState || mVideoDeviceState; }
bool Stopped() const { return mStopped; }
bool CapturingVideo() const;
bool CapturingAudio() const;
CaptureState CapturingSource(MediaSourceEnum aSource) const;
RefPtr<SourceListenerPromise> ApplyConstraintsToTrack(
MediaTrack* aTrack, const MediaTrackConstraints& aConstraints,
CallerType aCallerType);
PrincipalHandle GetPrincipalHandle() const;
private:
virtual ~SourceListener() = default;
using DeviceOperationPromise =
MozPromise<nsresult, bool, /* IsExclusive = */ true>;
/**
* Posts a task to start or stop the device associated with aTrack, based on
* a passed-in boolean. Private method used by SetEnabledFor and SetMutedFor.
*/
RefPtr<DeviceOperationPromise> UpdateDevice(MediaTrack* aTrack, bool aOn);
/**
* Returns a pointer to the device state for aTrack.
*
* This is intended for internal use where we need to figure out which state
* corresponds to aTrack, not for availability checks. As such, we assert
* that the device does indeed exist.
*
* Since this is a raw pointer and the state lifetime depends on the
* SourceListener's lifetime, it's internal use only.
*/
DeviceState& GetDeviceStateFor(MediaTrack* aTrack) const;
// true after this listener has had all devices stopped. MainThread only.
bool mStopped;
// never ever indirect off this; just for assertions
PRThread* mMainThreadCheck;
// Set in Register() on main thread, then read from any thread.
PrincipalHandle mPrincipalHandle;
// Weak pointer to the window listener that owns us. MainThread only.
GetUserMediaWindowListener* mWindowListener;
// Accessed from MediaTrackGraph thread, MediaManager thread, and MainThread
// No locking needed as they're set on Activate() and never assigned to again.
UniquePtr<DeviceState> mAudioDeviceState;
UniquePtr<DeviceState> mVideoDeviceState;
};
/**
* This class represents a WindowID and handles all MediaTrackListeners
* (here subclassed as SourceListeners) used to feed GetUserMedia source
* streams. It proxies feedback from them into messages for browser chrome.
* The SourceListeners are used to Start() and Stop() the underlying
* MediaEngineSource when MediaStreams are assigned and deassigned in content.
*/
class GetUserMediaWindowListener {
friend MediaManager;
public:
NS_INLINE_DECL_THREADSAFE_REFCOUNTING(GetUserMediaWindowListener)
// Create in an inactive state
GetUserMediaWindowListener(uint64_t aWindowID,
const PrincipalHandle& aPrincipalHandle)
: mWindowID(aWindowID),
mPrincipalHandle(aPrincipalHandle),
mChromeNotificationTaskPosted(false) {}
/**
* Registers an inactive gUM source listener for this WindowListener.
*/
void Register(RefPtr<SourceListener> aListener) {
MOZ_ASSERT(NS_IsMainThread());
MOZ_ASSERT(aListener);
MOZ_ASSERT(!aListener->Activated());
MOZ_ASSERT(!mInactiveListeners.Contains(aListener), "Already registered");
MOZ_ASSERT(!mActiveListeners.Contains(aListener), "Already activated");
aListener->Register(this);
mInactiveListeners.AppendElement(std::move(aListener));
}
/**
* Activates an already registered and inactive gUM source listener for this
* WindowListener.
*/
void Activate(RefPtr<SourceListener> aListener,
RefPtr<MediaDevice> aAudioDevice,
RefPtr<LocalTrackSource> aAudioTrackSource,
RefPtr<MediaDevice> aVideoDevice,
RefPtr<LocalTrackSource> aVideoTrackSource) {
MOZ_ASSERT(NS_IsMainThread());
MOZ_ASSERT(aListener);
MOZ_ASSERT(!aListener->Activated());
MOZ_ASSERT(mInactiveListeners.Contains(aListener),
"Must be registered to activate");
MOZ_ASSERT(!mActiveListeners.Contains(aListener), "Already activated");
mInactiveListeners.RemoveElement(aListener);
aListener->Activate(std::move(aAudioDevice), std::move(aAudioTrackSource),
std::move(aVideoDevice), std::move(aVideoTrackSource),
mCamerasAreMuted, mMicrophonesAreMuted);
mActiveListeners.AppendElement(std::move(aListener));
}
/**
* Removes all SourceListeners from this window listener.
* Removes this window listener from the list of active windows, so callers
* need to make sure to hold a strong reference.
*/
void RemoveAll() {
MOZ_ASSERT(NS_IsMainThread());
for (auto& l : mInactiveListeners.Clone()) {
Remove(l);
}
for (auto& l : mActiveListeners.Clone()) {
Remove(l);
}
MOZ_ASSERT(mInactiveListeners.Length() == 0);
MOZ_ASSERT(mActiveListeners.Length() == 0);
MediaManager* mgr = MediaManager::GetIfExists();
if (!mgr) {
MOZ_ASSERT(false, "MediaManager should stay until everything is removed");
return;
}
GetUserMediaWindowListener* windowListener =
mgr->GetWindowListener(mWindowID);
if (!windowListener) {
nsCOMPtr<nsIObserverService> obs = services::GetObserverService();
auto* globalWindow = nsGlobalWindowInner::GetInnerWindowWithId(mWindowID);
if (globalWindow) {
auto req = MakeRefPtr<GetUserMediaRequest>(
globalWindow, VoidString(), VoidString(),
UserActivation::IsHandlingUserInput());
obs->NotifyWhenScriptSafe(req, "recording-device-stopped", nullptr);
}
return;
}
MOZ_ASSERT(windowListener == this,
"There should only be one window listener per window ID");
LOG("GUMWindowListener %p removing windowID %" PRIu64, this, mWindowID);
mgr->RemoveWindowID(mWindowID);
}
/**
* Removes a listener from our lists. Safe to call without holding a hard
* reference. That said, you'll still want to iterate on a copy of said lists,
* if you end up calling this method (or methods that may call this method) in
* the loop, to avoid inadvertently skipping members.
*/
bool Remove(RefPtr<SourceListener> aListener) {
// We refcount aListener on entry since we're going to proxy-release it
// below to prevent the refcount going to zero on callers who might be
// inside the listener, but operating without a hard reference to self.
MOZ_ASSERT(NS_IsMainThread());
if (!mInactiveListeners.RemoveElement(aListener) &&
!mActiveListeners.RemoveElement(aListener)) {
return false;
}
MOZ_ASSERT(!mInactiveListeners.Contains(aListener),
"A SourceListener should only be once in one of "
"mInactiveListeners and mActiveListeners");
MOZ_ASSERT(!mActiveListeners.Contains(aListener),
"A SourceListener should only be once in one of "
"mInactiveListeners and mActiveListeners");
LOG("GUMWindowListener %p stopping SourceListener %p.", this,
aListener.get());
aListener->Stop();
if (MediaDevice* removedDevice = aListener->GetVideoDevice()) {
bool revokeVideoPermission = true;
nsString removedRawId;
nsString removedSourceType;
removedDevice->GetRawId(removedRawId);
removedDevice->GetMediaSource(removedSourceType);
for (const auto& l : mActiveListeners) {
if (MediaDevice* device = l->GetVideoDevice()) {
nsString rawId;
device->GetRawId(rawId);
if (removedRawId.Equals(rawId)) {
revokeVideoPermission = false;
break;
}
}
}
if (revokeVideoPermission) {
nsCOMPtr<nsIObserverService> obs = services::GetObserverService();
auto* window = nsGlobalWindowInner::GetInnerWindowWithId(mWindowID);
auto req = MakeRefPtr<GetUserMediaRequest>(
window, removedRawId, removedSourceType,
UserActivation::IsHandlingUserInput());
obs->NotifyWhenScriptSafe(req, "recording-device-stopped", nullptr);
}
}
if (MediaDevice* removedDevice = aListener->GetAudioDevice()) {
bool revokeAudioPermission = true;
nsString removedRawId;
nsString removedSourceType;
removedDevice->GetRawId(removedRawId);
removedDevice->GetMediaSource(removedSourceType);
for (const auto& l : mActiveListeners) {
if (MediaDevice* device = l->GetAudioDevice()) {
nsString rawId;
device->GetRawId(rawId);
if (removedRawId.Equals(rawId)) {
revokeAudioPermission = false;
break;
}
}
}
if (revokeAudioPermission) {
nsCOMPtr<nsIObserverService> obs = services::GetObserverService();
auto* window = nsGlobalWindowInner::GetInnerWindowWithId(mWindowID);
auto req = MakeRefPtr<GetUserMediaRequest>(
window, removedRawId, removedSourceType,
UserActivation::IsHandlingUserInput());
obs->NotifyWhenScriptSafe(req, "recording-device-stopped", nullptr);
}
}
if (mInactiveListeners.Length() == 0 && mActiveListeners.Length() == 0) {
LOG("GUMWindowListener %p Removed last SourceListener. Cleaning up.",
this);
RemoveAll();
}
nsCOMPtr<nsIEventTarget> mainTarget = do_GetMainThread();
// To allow being invoked by callers not holding a strong reference to self,
// hold the listener alive until the stack has unwound, by always
// dispatching a runnable (aAlwaysProxy = true)
NS_ProxyRelease(__func__, mainTarget, aListener.forget(), true);
return true;
}
void StopSharing();
void StopRawID(const nsString& removedDeviceID);
void MuteOrUnmuteCameras(bool aMute);
void MuteOrUnmuteMicrophones(bool aMute);
/**
* Called by one of our SourceListeners when one of its tracks has changed so
* that chrome state is affected.
* Schedules an event for the next stable state to update chrome.
*/
void ChromeAffectingStateChanged();
/**
* Called in stable state to send a notification to update chrome.
*/
void NotifyChrome();
bool CapturingVideo() const {
MOZ_ASSERT(NS_IsMainThread());
for (auto& l : mActiveListeners) {
if (l->CapturingVideo()) {
return true;
}
}
return false;
}
bool CapturingAudio() const {
MOZ_ASSERT(NS_IsMainThread());
for (auto& l : mActiveListeners) {
if (l->CapturingAudio()) {
return true;
}
}
return false;
}
CaptureState CapturingSource(MediaSourceEnum aSource) const {
MOZ_ASSERT(NS_IsMainThread());
CaptureState result = CaptureState::Off;
for (auto& l : mActiveListeners) {
result = CombineCaptureState(result, l->CapturingSource(aSource));
}
return result;
}
void GetDevices(
const RefPtr<MediaManager::MediaDeviceSetRefCnt>& aOutDevices) {
for (auto& l : mActiveListeners) {
if (RefPtr<MediaDevice> device = l->GetAudioDevice()) {
aOutDevices->AppendElement(device);
} else if (RefPtr<MediaDevice> device = l->GetVideoDevice()) {
aOutDevices->AppendElement(device);
}
}
}
uint64_t WindowID() const { return mWindowID; }
PrincipalHandle GetPrincipalHandle() const { return mPrincipalHandle; }
private:
~GetUserMediaWindowListener() {
MOZ_ASSERT(mInactiveListeners.Length() == 0,
"Inactive listeners should already be removed");
MOZ_ASSERT(mActiveListeners.Length() == 0,
"Active listeners should already be removed");
}
uint64_t mWindowID;
const PrincipalHandle mPrincipalHandle;
// true if we have scheduled a task to notify chrome in the next stable state.
// The task will reset this to false. MainThread only.
bool mChromeNotificationTaskPosted;
nsTArray<RefPtr<SourceListener>> mInactiveListeners;
nsTArray<RefPtr<SourceListener>> mActiveListeners;
// Whether camera and microphone access in this window are currently
// User Agent (UA) muted. When true, new and cloned tracks must start
// out muted, to avoid JS circumventing UA mute. Per-camera and
// per-microphone UA muting is not supported.
bool mCamerasAreMuted = false;
bool mMicrophonesAreMuted = false;
};
class LocalTrackSource : public MediaStreamTrackSource {
public:
LocalTrackSource(nsIPrincipal* aPrincipal, const nsString& aLabel,
const RefPtr<SourceListener>& aListener,
MediaSourceEnum aSource, MediaTrack* aTrack,
RefPtr<PeerIdentity> aPeerIdentity)
: MediaStreamTrackSource(aPrincipal, aLabel),
mSource(aSource),
mTrack(aTrack),
mPeerIdentity(std::move(aPeerIdentity)),
mListener(aListener.get()) {}
MediaSourceEnum GetMediaSource() const override { return mSource; }
const PeerIdentity* GetPeerIdentity() const override { return mPeerIdentity; }
RefPtr<MediaStreamTrackSource::ApplyConstraintsPromise> ApplyConstraints(
const MediaTrackConstraints& aConstraints,
CallerType aCallerType) override {
MOZ_ASSERT(NS_IsMainThread());
if (sHasShutdown || !mListener) {
// Track has been stopped, or we are in shutdown. In either case
// there's no observable outcome, so pretend we succeeded.
return MediaStreamTrackSource::ApplyConstraintsPromise::CreateAndResolve(
false, __func__);
}
return mListener->ApplyConstraintsToTrack(mTrack, aConstraints,
aCallerType);
}
void GetSettings(MediaTrackSettings& aOutSettings) override {
if (mListener) {
mListener->GetSettingsFor(mTrack, aOutSettings);
}
}
void Stop() override {
if (mListener) {
mListener->StopTrack(mTrack);
mListener = nullptr;
}
if (!mTrack->IsDestroyed()) {
mTrack->Destroy();
}
}
void Disable() override {
if (mListener) {
mListener->SetEnabledFor(mTrack, false);
}
}
void Enable() override {
if (mListener) {
mListener->SetEnabledFor(mTrack, true);
}
}
void Mute() {
MutedChanged(true);
mTrack->SetDisabledTrackMode(DisabledTrackMode::SILENCE_BLACK);
}
void Unmute() {
MutedChanged(false);
mTrack->SetDisabledTrackMode(DisabledTrackMode::ENABLED);
}
const MediaSourceEnum mSource;
const RefPtr<MediaTrack> mTrack;
const RefPtr<const PeerIdentity> mPeerIdentity;
protected:
~LocalTrackSource() {
MOZ_ASSERT(NS_IsMainThread());
MOZ_ASSERT(mTrack->IsDestroyed());
}
// This is a weak pointer to avoid having the SourceListener (which may
// have references to threads and threadpools) kept alive by DOM-objects
// that may have ref-cycles and thus are released very late during
// shutdown, even after xpcom-shutdown-threads. See bug 1351655 for what
// can happen.
WeakPtr<SourceListener> mListener;
};
class AudioCaptureTrackSource : public LocalTrackSource {
public:
AudioCaptureTrackSource(nsIPrincipal* aPrincipal, nsPIDOMWindowInner* aWindow,
const nsString& aLabel,
AudioCaptureTrack* aAudioCaptureTrack,
RefPtr<PeerIdentity> aPeerIdentity)
: LocalTrackSource(aPrincipal, aLabel, nullptr,
MediaSourceEnum::AudioCapture, aAudioCaptureTrack,
std::move(aPeerIdentity)),
mWindow(aWindow),
mAudioCaptureTrack(aAudioCaptureTrack) {
mAudioCaptureTrack->Start();
mAudioCaptureTrack->Graph()->RegisterCaptureTrackForWindow(
mWindow->WindowID(), mAudioCaptureTrack);
mWindow->SetAudioCapture(true);
}
void Stop() override {
MOZ_ASSERT(NS_IsMainThread());
if (!mAudioCaptureTrack->IsDestroyed()) {
MOZ_ASSERT(mWindow);
mWindow->SetAudioCapture(false);
mAudioCaptureTrack->Graph()->UnregisterCaptureTrackForWindow(
mWindow->WindowID());
mWindow = nullptr;
}
// LocalTrackSource destroys the track.
LocalTrackSource::Stop();
MOZ_ASSERT(mAudioCaptureTrack->IsDestroyed());
}
ProcessedMediaTrack* InputTrack() const { return mAudioCaptureTrack.get(); }
protected:
~AudioCaptureTrackSource() {
MOZ_ASSERT(NS_IsMainThread());
MOZ_ASSERT(mAudioCaptureTrack->IsDestroyed());
}
RefPtr<nsPIDOMWindowInner> mWindow;
const RefPtr<AudioCaptureTrack> mAudioCaptureTrack;
};
/**
* nsIMediaDevice implementation.
*/
NS_IMPL_ISUPPORTS(MediaDevice, nsIMediaDevice)
MediaDevice::MediaDevice(const RefPtr<MediaEngineSource>& aSource,
const nsString& aName, const nsString& aID,
const nsString& aGroupID, const nsString& aRawID)
: mSource(aSource),
mSinkInfo(nullptr),
mKind((mSource && MediaEngineSource::IsVideo(mSource->GetMediaSource()))
? MediaDeviceKind::Videoinput
: MediaDeviceKind::Audioinput),
mScary(mSource->GetScary()),
mIsFake(mSource->IsFake()),
mType(
NS_ConvertASCIItoUTF16(dom::MediaDeviceKindValues::GetString(mKind))),
mName(aName),
mID(aID),
mGroupID(aGroupID),
mRawID(aRawID),
mRawName(aName) {
MOZ_ASSERT(mSource);
}
MediaDevice::MediaDevice(const RefPtr<AudioDeviceInfo>& aAudioDeviceInfo,
const nsString& aID, const nsString& aGroupID,
const nsString& aRawID)
: mSource(nullptr),
mSinkInfo(aAudioDeviceInfo),
mKind(mSinkInfo->Type() == AudioDeviceInfo::TYPE_INPUT
? MediaDeviceKind::Audioinput
: MediaDeviceKind::Audiooutput),
mScary(false),
mIsFake(false),
mType(
NS_ConvertASCIItoUTF16(dom::MediaDeviceKindValues::GetString(mKind))),
mName(mSinkInfo->Name()),
mID(aID),
mGroupID(aGroupID),
mRawID(aRawID),
mRawName(mSinkInfo->Name()) {
// For now this ctor is used only for Audiooutput.
// It could be used for Audioinput and Videoinput
// when we do not instantiate a MediaEngineSource
// during EnumerateDevices.
MOZ_ASSERT(mKind == MediaDeviceKind::Audiooutput);
MOZ_ASSERT(mSinkInfo);
}
MediaDevice::MediaDevice(const RefPtr<MediaDevice>& aOther, const nsString& aID,
const nsString& aGroupID, const nsString& aRawID,
const nsString& aRawGroupID)
: MediaDevice(aOther, aID, aGroupID, aRawID, aRawGroupID, aOther->mName) {}
MediaDevice::MediaDevice(const RefPtr<MediaDevice>& aOther, const nsString& aID,
const nsString& aGroupID, const nsString& aRawID,
const nsString& aRawGroupID, const nsString& aName)
: mSource(aOther->mSource),
mSinkInfo(aOther->mSinkInfo),
mKind(aOther->mKind),
mScary(aOther->mScary),
mIsFake(aOther->mIsFake),
mType(aOther->mType),
mName(aName),
mID(aID),
mGroupID(aGroupID),
mRawID(aRawID),
mRawGroupID(aRawGroupID),
mRawName(aOther->mRawName) {
MOZ_ASSERT(aOther);
}
/**
* Helper functions that implement the constraints algorithm from
*/
/* static */
bool MediaDevice::StringsContain(const OwningStringOrStringSequence& aStrings,
nsString aN) {
return aStrings.IsString() ? aStrings.GetAsString() == aN
: aStrings.GetAsStringSequence().Contains(aN);
}
/* static */
uint32_t MediaDevice::FitnessDistance(
nsString aN, const ConstrainDOMStringParameters& aParams) {
if (aParams.mExact.WasPassed() &&
!StringsContain(aParams.mExact.Value(), aN)) {
return UINT32_MAX;
}
if (aParams.mIdeal.WasPassed() &&
!StringsContain(aParams.mIdeal.Value(), aN)) {
return 1;
}
return 0;
}
// Binding code doesn't templatize well...
/* static */
uint32_t MediaDevice::FitnessDistance(
nsString aN,
const OwningStringOrStringSequenceOrConstrainDOMStringParameters&
aConstraint) {
if (aConstraint.IsString()) {
ConstrainDOMStringParameters params;
params.mIdeal.Construct();
params.mIdeal.Value().SetAsString() = aConstraint.GetAsString();
return FitnessDistance(aN, params);
} else if (aConstraint.IsStringSequence()) {
ConstrainDOMStringParameters params;
params.mIdeal.Construct();
params.mIdeal.Value().SetAsStringSequence() =
aConstraint.GetAsStringSequence();
return FitnessDistance(aN, params);
} else {
return FitnessDistance(aN, aConstraint.GetAsConstrainDOMStringParameters());
}
}
uint32_t MediaDevice::GetBestFitnessDistance(
const nsTArray<const NormalizedConstraintSet*>& aConstraintSets,
bool aIsChrome) {
MOZ_ASSERT(MediaManager::IsInMediaThread());
MOZ_ASSERT(mSource);
const nsString& id = aIsChrome ? mRawID : mID;
auto type = GetMediaSource();
uint64_t distance = 0;
if (!aConstraintSets.IsEmpty()) {
if (aIsChrome /* For the screen/window sharing preview */ ||
type == MediaSourceEnum::Camera ||
type == MediaSourceEnum::Microphone) {
distance += uint64_t(MediaConstraintsHelper::FitnessDistance(
Some(id), aConstraintSets[0]->mDeviceId)) +
uint64_t(MediaConstraintsHelper::FitnessDistance(
Some(mGroupID), aConstraintSets[0]->mGroupId));
}
}
if (distance < UINT32_MAX) {
// Forward request to underlying object to interrogate per-mode
// capabilities.
distance += mSource->GetBestFitnessDistance(aConstraintSets);
}
return std::min<uint64_t>(distance, UINT32_MAX);
}
NS_IMETHODIMP
MediaDevice::GetName(nsAString& aName) {
MOZ_ASSERT(NS_IsMainThread());
aName.Assign(mName);
return NS_OK;
}
NS_IMETHODIMP
MediaDevice::GetRawName(nsAString& aName) {
MOZ_ASSERT(NS_IsMainThread());
aName.Assign(mRawName);
return NS_OK;
}
NS_IMETHODIMP
MediaDevice::GetType(nsAString& aType) {
MOZ_ASSERT(NS_IsMainThread());
aType.Assign(mType);
return NS_OK;
}
NS_IMETHODIMP
MediaDevice::GetId(nsAString& aID) {
MOZ_ASSERT(NS_IsMainThread());
aID.Assign(mID);
return NS_OK;
}
NS_IMETHODIMP
MediaDevice::GetRawId(nsAString& aID) {
MOZ_ASSERT(NS_IsMainThread());
aID.Assign(mRawID);
return NS_OK;
}
NS_IMETHODIMP
MediaDevice::GetGroupId(nsAString& aGroupID) {
MOZ_ASSERT(NS_IsMainThread());
aGroupID.Assign(mGroupID);
return NS_OK;
}
NS_IMETHODIMP
MediaDevice::GetRawGroupId(nsAString& aRawGroupID) {
MOZ_ASSERT(NS_IsMainThread());
aRawGroupID.Assign(mRawGroupID);
return NS_OK;
}
NS_IMETHODIMP
MediaDevice::GetScary(bool* aScary) {
*aScary = mScary;
return NS_OK;
}
void MediaDevice::GetSettings(MediaTrackSettings& aOutSettings) const {
MOZ_ASSERT(NS_IsMainThread());
MOZ_ASSERT(mSource);
mSource->GetSettings(aOutSettings);
}
// Threadsafe since mSource is const.
NS_IMETHODIMP
MediaDevice::GetMediaSource(nsAString& aMediaSource) {
aMediaSource.AssignASCII(
dom::MediaSourceEnumValues::GetString(GetMediaSource()));
return NS_OK;
}
nsresult MediaDevice::Allocate(const MediaTrackConstraints& aConstraints,
const MediaEnginePrefs& aPrefs,
uint64_t aWindowID,
const char** aOutBadConstraint) {
MOZ_ASSERT(MediaManager::IsInMediaThread());
MOZ_ASSERT(mSource);
// Mock failure for automated tests.
if (mIsFake && aConstraints.mDeviceId.WasPassed() &&
aConstraints.mDeviceId.Value().IsString() &&
aConstraints.mDeviceId.Value().GetAsString().EqualsASCII("bad device")) {
return NS_ERROR_FAILURE;
}
return mSource->Allocate(aConstraints, aPrefs, aWindowID, aOutBadConstraint);
}
void MediaDevice::SetTrack(const RefPtr<SourceMediaTrack>& aTrack,
const PrincipalHandle& aPrincipalHandle) {
MOZ_ASSERT(MediaManager::IsInMediaThread());
MOZ_ASSERT(mSource);
mSource->SetTrack(aTrack, aPrincipalHandle);
}
nsresult MediaDevice::Start() {
MOZ_ASSERT(MediaManager::IsInMediaThread());
MOZ_ASSERT(mSource);
return mSource->Start();
}
nsresult MediaDevice::Reconfigure(const MediaTrackConstraints& aConstraints,
const MediaEnginePrefs& aPrefs,
const char** aOutBadConstraint) {
MOZ_ASSERT(MediaManager::IsInMediaThread());
MOZ_ASSERT(mSource);
auto type = GetMediaSource();
if (type == MediaSourceEnum::Camera || type == MediaSourceEnum::Microphone) {
NormalizedConstraints c(aConstraints);
if (MediaConstraintsHelper::FitnessDistance(Some(mID), c.mDeviceId) ==
UINT32_MAX) {
*aOutBadConstraint = "deviceId";
return NS_ERROR_INVALID_ARG;
}
if (MediaConstraintsHelper::FitnessDistance(Some(mGroupID), c.mGroupId) ==
UINT32_MAX) {
*aOutBadConstraint = "groupId";
return NS_ERROR_INVALID_ARG;
}
}
return mSource->Reconfigure(aConstraints, aPrefs, aOutBadConstraint);
}
nsresult MediaDevice::FocusOnSelectedSource() {
MOZ_ASSERT(MediaManager::IsInMediaThread());
MOZ_ASSERT(mSource);
return mSource->FocusOnSelectedSource();
}
nsresult MediaDevice::Stop() {
MOZ_ASSERT(MediaManager::IsInMediaThread());
MOZ_ASSERT(mSource);
return mSource->Stop();
}
nsresult MediaDevice::Deallocate() {
MOZ_ASSERT(MediaManager::IsInMediaThread());
MOZ_ASSERT(mSource);
return mSource->Deallocate();
}
MediaSourceEnum MediaDevice::GetMediaSource() const {
// Threadsafe because mSource is const. GetMediaSource() might have other
// requirements.
MOZ_ASSERT(mSource);
return mSource->GetMediaSource();
}
static bool IsOn(const OwningBooleanOrMediaTrackConstraints& aUnion) {
return !aUnion.IsBoolean() || aUnion.GetAsBoolean();
}
static const MediaTrackConstraints& GetInvariant(
const OwningBooleanOrMediaTrackConstraints& aUnion) {
static const MediaTrackConstraints empty;
return aUnion.IsMediaTrackConstraints() ? aUnion.GetAsMediaTrackConstraints()
: empty;
}
/**
* Creates a MediaTrack, attaches a listener and fires off a success callback
* to the DOM with the stream. We also pass in the error callback so it can
* be released correctly.
*
* All of this must be done on the main thread!
*
* Note that the various GetUserMedia Runnable classes currently allow for
* two tracks. If we ever need to support getting more than two tracks
* at once, we could convert everything to nsTArray<RefPtr<blah> >'s,
* though that would complicate the constructors some. Currently the
* GetUserMedia spec does not allow for more than 2 tracks to be obtained in
* one call, to simplify handling of constraints.
*/
class GetUserMediaStreamRunnable : public Runnable {
public:
GetUserMediaStreamRunnable(
MozPromiseHolder<MediaManager::StreamPromise>&& aHolder,
uint64_t aWindowID, RefPtr<GetUserMediaWindowListener> aWindowListener,
RefPtr<SourceListener> aSourceListener,
const ipc::PrincipalInfo& aPrincipalInfo,
const MediaStreamConstraints& aConstraints,
RefPtr<MediaDevice> aAudioDevice, RefPtr<MediaDevice> aVideoDevice,
RefPtr<PeerIdentity> aPeerIdentity, bool aIsChrome)
: Runnable("GetUserMediaStreamRunnable"),
mHolder(std::move(aHolder)),
mConstraints(aConstraints),
mAudioDevice(std::move(aAudioDevice)),
mVideoDevice(std::move(aVideoDevice)),
mWindowID(aWindowID),
mWindowListener(std::move(aWindowListener)),
mSourceListener(std::move(aSourceListener)),
mPrincipalInfo(aPrincipalInfo),
mPeerIdentity(std::move(aPeerIdentity)),
mManager(MediaManager::GetInstance()) {}
~GetUserMediaStreamRunnable() {
mHolder.RejectIfExists(
MakeRefPtr<MediaMgrError>(MediaMgrError::Name::AbortError), __func__);
}
NS_IMETHOD
Run() override {
MOZ_ASSERT(NS_IsMainThread());
LOG("GetUserMediaStreamRunnable::Run()");
nsGlobalWindowInner* window =
nsGlobalWindowInner::GetInnerWindowWithId(mWindowID);
// We're on main-thread, and the windowlist can only
// be invalidated from the main-thread (see OnNavigation)
if (!mManager->IsWindowListenerStillActive(mWindowListener)) {
// This window is no longer live. mListener has already been removed.
return NS_OK;
}
MediaTrackGraph::GraphDriverType graphDriverType =
mAudioDevice ? MediaTrackGraph::AUDIO_THREAD_DRIVER
: MediaTrackGraph::SYSTEM_THREAD_DRIVER;
MediaTrackGraph* mtg = MediaTrackGraph::GetInstance(
graphDriverType, window, MediaTrackGraph::REQUEST_DEFAULT_SAMPLE_RATE,
MediaTrackGraph::DEFAULT_OUTPUT_DEVICE);
auto domStream = MakeRefPtr<DOMMediaStream>(window);
RefPtr<LocalTrackSource> audioTrackSource;
RefPtr<LocalTrackSource> videoTrackSource;
nsCOMPtr<nsIPrincipal> principal;
if (mPeerIdentity) {
principal = NullPrincipal::CreateWithInheritedAttributes(
window->GetExtantDoc()->NodePrincipal());
} else {
principal = window->GetExtantDoc()->NodePrincipal();
}
RefPtr<GenericNonExclusivePromise> firstFramePromise;
if (mAudioDevice) {
if (mAudioDevice->GetMediaSource() == MediaSourceEnum::AudioCapture) {
// AudioCapture is a special case, here, in the sense that we're not
// really using the audio source and the SourceMediaTrack, which acts
// as placeholders. We re-route a number of tracks internally in the
// MTG and mix them down instead.
NS_WARNING(
"MediaCaptureWindowState doesn't handle "
"MediaSourceEnum::AudioCapture. This must be fixed with UX "
"before shipping.");
auto audioCaptureSource = MakeRefPtr<AudioCaptureTrackSource>(
principal, window, u"Window audio capture"_ns,
mtg->CreateAudioCaptureTrack(), mPeerIdentity);
audioTrackSource = audioCaptureSource;
RefPtr<MediaStreamTrack> track = new dom::AudioStreamTrack(
window, audioCaptureSource->InputTrack(), audioCaptureSource);
domStream->AddTrackInternal(track);
} else {
nsString audioDeviceName;
mAudioDevice->GetName(audioDeviceName);
RefPtr<MediaTrack> track = mtg->CreateSourceTrack(MediaSegment::AUDIO);
audioTrackSource = new LocalTrackSource(
principal, audioDeviceName, mSourceListener,
mAudioDevice->GetMediaSource(), track, mPeerIdentity);
MOZ_ASSERT(IsOn(mConstraints.mAudio));
RefPtr<MediaStreamTrack> domTrack = new dom::AudioStreamTrack(
window, track, audioTrackSource, dom::MediaStreamTrackState::Live,
false, GetInvariant(mConstraints.mAudio));
domStream->AddTrackInternal(domTrack);
}
}
if (mVideoDevice) {
nsString videoDeviceName;
mVideoDevice->GetName(videoDeviceName);
RefPtr<MediaTrack> track = mtg->CreateSourceTrack(MediaSegment::VIDEO);
videoTrackSource = new LocalTrackSource(
principal, videoDeviceName, mSourceListener,
mVideoDevice->GetMediaSource(), track, mPeerIdentity);
MOZ_ASSERT(IsOn(mConstraints.mVideo));
RefPtr<MediaStreamTrack> domTrack = new dom::VideoStreamTrack(
window, track, videoTrackSource, dom::MediaStreamTrackState::Live,
false, GetInvariant(mConstraints.mVideo));
domStream->AddTrackInternal(domTrack);
switch (mVideoDevice->GetMediaSource()) {
case MediaSourceEnum::Browser:
case MediaSourceEnum::Screen:
case MediaSourceEnum::Window:
// Wait for first frame for screen-sharing devices, to ensure
// with and height settings are available immediately, to pass wpt.
firstFramePromise = mVideoDevice->mSource->GetFirstFramePromise();
break;
default:
break;
}
}
if (!domStream || (!audioTrackSource && !videoTrackSource) ||
sHasShutdown) {
LOG("Returning error for getUserMedia() - no stream");
mHolder.Reject(MakeRefPtr<MediaMgrError>(
MediaMgrError::Name::AbortError,
sHasShutdown ? u"In shutdown"_ns : u"No stream."_ns),
__func__);
return NS_OK;
}
// Activate our source listener. We'll call Start() on the source when we
// get a callback that the MediaStream has started consuming. The listener
// is freed when the page is invalidated (on navigation or close).
mWindowListener->Activate(mSourceListener, mAudioDevice,
std::move(audioTrackSource), mVideoDevice,
std::move(videoTrackSource));
// Dispatch to the media thread to ask it to start the sources, because that
// can take a while.
mSourceListener->InitializeAsync()
->Then(
GetMainThreadSerialEventTarget(), __func__,
[manager = mManager, windowListener = mWindowListener,
firstFramePromise] {
LOG("GetUserMediaStreamRunnable::Run: starting success callback "
"following InitializeAsync()");
// Initiating and starting devices succeeded.
windowListener->ChromeAffectingStateChanged();
manager->SendPendingGUMRequest();
if (!firstFramePromise) {
return SourceListener::SourceListenerPromise::CreateAndResolve(
true, __func__);
}
RefPtr<SourceListener::SourceListenerPromise> resolvePromise =
firstFramePromise->Then(
GetMainThreadSerialEventTarget(), __func__,
[] {
return SourceListener::SourceListenerPromise::
CreateAndResolve(true, __func__);
},
[] {
return SourceListener::SourceListenerPromise::
CreateAndReject(MakeRefPtr<MediaMgrError>(
MediaMgrError::Name::AbortError,
u"In shutdown"_ns),
__func__);
});
return resolvePromise;
},
[](RefPtr<MediaMgrError>&& aError) {
LOG("GetUserMediaStreamRunnable::Run: starting failure callback "
"following InitializeAsync()");
return SourceListener::SourceListenerPromise::CreateAndReject(
aError, __func__);
})
->Then(GetMainThreadSerialEventTarget(), __func__,
[holder = std::move(mHolder),
domStream](const SourceListener::SourceListenerPromise::
ResolveOrRejectValue& aValue) mutable {
if (aValue.IsResolve()) {
holder.Resolve(domStream, __func__);
} else {
holder.Reject(aValue.RejectValue(), __func__);
}
});
if (!IsPrincipalInfoPrivate(mPrincipalInfo)) {
// Call GetPrincipalKey again, this time w/persist = true, to promote
// deviceIds to persistent, in case they're not already. Fire'n'forget.
media::GetPrincipalKey(mPrincipalInfo, true)
->Then(GetCurrentSerialEventTarget(), __func__,
[](const media::PrincipalKeyPromise::ResolveOrRejectValue&
aValue) {
if (aValue.IsReject()) {
LOG("Failed get Principal key. Persisting of deviceIds "
"will be broken");
}
});
}
return NS_OK;
}
private:
MozPromiseHolder<MediaManager::StreamPromise> mHolder;
MediaStreamConstraints mConstraints;
RefPtr<MediaDevice> mAudioDevice;
RefPtr<MediaDevice> mVideoDevice;
uint64_t mWindowID;
RefPtr<GetUserMediaWindowListener> mWindowListener;
RefPtr<SourceListener> mSourceListener;
ipc::PrincipalInfo mPrincipalInfo;
RefPtr<PeerIdentity> mPeerIdentity;
RefPtr<MediaManager> mManager; // get ref to this when creating the runnable
};
// Source getter returning full list
static void GetMediaDevices(MediaEngine* aEngine, uint64_t aWindowId,
MediaSourceEnum aSrcType,
MediaManager::MediaDeviceSet& aResult,
const char* aMediaDeviceName = nullptr) {
MOZ_ASSERT(MediaManager::IsInMediaThread());
LOG("%s: aEngine=%p, aWindowId=%" PRIu64 ", aSrcType=%" PRIu8
", aMediaDeviceName=%s",
__func__, aEngine, aWindowId, static_cast<uint8_t>(aSrcType),
aMediaDeviceName ? aMediaDeviceName : "null");
nsTArray<RefPtr<MediaDevice>> devices;
aEngine->EnumerateDevices(aWindowId, aSrcType, MediaSinkEnum::Other,
&devices);
/*
* We're allowing multiple tabs to access the same camera for parity
* with Chrome. See bug 811757 for some of the issues surrounding
* this decision. To disallow, we'd filter by IsAvailable() as we used
* to.
*/
if (aMediaDeviceName && *aMediaDeviceName) {
for (auto& device : devices) {
if (device->mName.EqualsASCII(aMediaDeviceName)) {
aResult.AppendElement(device);
LOG("%s: found aMediaDeviceName=%s", __func__, aMediaDeviceName);
break;
}
}
} else {
aResult = std::move(devices);
if (MOZ_LOG_TEST(gMediaManagerLog, mozilla::LogLevel::Debug)) {
for (auto& device : aResult) {
LOG("%s: appending device=%s", __func__,
NS_ConvertUTF16toUTF8(device->mName).get());
}
}
}
}
RefPtr<MediaManager::BadConstraintsPromise> MediaManager::SelectSettings(
const MediaStreamConstraints& aConstraints, bool aIsChrome,
const RefPtr<MediaDeviceSetRefCnt>& aSources) {
MOZ_ASSERT(NS_IsMainThread());
// Algorithm accesses device capabilities code and must run on media thread.
// Modifies passed-in aSources.
return MediaManager::Dispatch<BadConstraintsPromise>(
__func__, [aConstraints, aSources,
aIsChrome](MozPromiseHolder<BadConstraintsPromise>& holder) {
auto& sources = *aSources;
// Since the advanced part of the constraints algorithm needs to know
// when a candidate set is overconstrained (zero members), we must split
// up the list into videos and audios, and put it back together again at
// the end.
nsTArray<RefPtr<MediaDevice>> videos;
nsTArray<RefPtr<MediaDevice>> audios;
for (auto& source : sources) {
MOZ_ASSERT(source->mKind == MediaDeviceKind::Videoinput ||
source->mKind == MediaDeviceKind::Audioinput);
if (source->mKind == MediaDeviceKind::Videoinput) {
videos.AppendElement(source);
} else if (source->mKind == MediaDeviceKind::Audioinput) {
audios.AppendElement(source);
}
}
sources.Clear();
const char* badConstraint = nullptr;
bool needVideo = IsOn(aConstraints.mVideo);
bool needAudio = IsOn(aConstraints.mAudio);
if (needVideo && videos.Length()) {
badConstraint = MediaConstraintsHelper::SelectSettings(
NormalizedConstraints(GetInvariant(aConstraints.mVideo)), videos,
aIsChrome);
}
if (!badConstraint && needAudio && audios.Length()) {
badConstraint = MediaConstraintsHelper::SelectSettings(
NormalizedConstraints(GetInvariant(aConstraints.mAudio)), audios,
aIsChrome);
}
if (!badConstraint && !needVideo == !videos.Length() &&
!needAudio == !audios.Length()) {
for (auto& video : videos) {
sources.AppendElement(video);
}
for (auto& audio : audios) {
sources.AppendElement(audio);
}
}
holder.Resolve(badConstraint, __func__);
});
}
/**
* Runs on a seperate thread and is responsible for enumerating devices.
* Depending on whether a picture or stream was asked for, either
* ProcessGetUserMedia is called, and the results are sent back to the DOM.
*
* Do not run this on the main thread. The success and error callbacks *MUST*
* be dispatched on the main thread!
*/
class GetUserMediaTask : public Runnable {
public:
GetUserMediaTask(const MediaStreamConstraints& aConstraints,
MozPromiseHolder<MediaManager::StreamPromise>&& aHolder,
uint64_t aWindowID,
RefPtr<GetUserMediaWindowListener> aWindowListener,
RefPtr<SourceListener> aSourceListener,
const MediaEnginePrefs& aPrefs,
const ipc::PrincipalInfo& aPrincipalInfo, bool aIsChrome,
RefPtr<MediaManager::MediaDeviceSetRefCnt>&& aMediaDeviceSet,
bool aShouldFocusSource)
: Runnable("GetUserMediaTask"),
mConstraints(aConstraints),
mHolder(std::move(aHolder)),
mWindowID(aWindowID),
mWindowListener(std::move(aWindowListener)),
mSourceListener(std::move(aSourceListener)),
mPrefs(aPrefs),
mPrincipalInfo(aPrincipalInfo),
mIsChrome(aIsChrome),
mShouldFocusSource(aShouldFocusSource),
mDeviceChosen(false),
mMediaDeviceSet(aMediaDeviceSet),
mManager(MediaManager::GetInstance()) {}
~GetUserMediaTask() {
if (!mHolder.IsEmpty()) {
Fail(MediaMgrError::Name::NotAllowedError);
}
}
void Fail(MediaMgrError::Name aName, const nsString& aMessage = u""_ns,
const nsString& aConstraint = u""_ns) {
NS_DispatchToMainThread(NS_NewRunnableFunction(
"GetUserMediaTask::Fail",
[aName, aMessage, aConstraint, holder = std::move(mHolder)]() mutable {
holder.Reject(MakeRefPtr<MediaMgrError>(aName, aMessage, aConstraint),
__func__);
}));
// Do after the above runs, as it checks active window list
NS_DispatchToMainThread(NewRunnableMethod<RefPtr<SourceListener>>(
"GetUserMediaWindowListener::Remove", mWindowListener,
&GetUserMediaWindowListener::Remove, mSourceListener));
}
NS_IMETHOD
Run() override {
MOZ_ASSERT(!NS_IsMainThread());
MOZ_ASSERT(mDeviceChosen);
LOG("GetUserMediaTask::Run()");
// Allocate a video or audio device and return a MediaStream via
// a GetUserMediaStreamRunnable.
nsresult rv;
const char* errorMsg = nullptr;
const char* badConstraint = nullptr;
if (mAudioDevice) {
auto& constraints = GetInvariant(mConstraints.mAudio);
rv = mAudioDevice->Allocate(constraints, mPrefs, mWindowID,
&badConstraint);
if (NS_FAILED(rv)) {
errorMsg = "Failed to allocate audiosource";
if (rv == NS_ERROR_NOT_AVAILABLE && !badConstraint) {
nsTArray<RefPtr<MediaDevice>> devices;
devices.AppendElement(mAudioDevice);
badConstraint = MediaConstraintsHelper::SelectSettings(
NormalizedConstraints(constraints), devices, mIsChrome);
}
}
}
if (!errorMsg && mVideoDevice) {
auto& constraints = GetInvariant(mConstraints.mVideo);
rv = mVideoDevice->Allocate(constraints, mPrefs, mWindowID,
&badConstraint);
if (NS_FAILED(rv)) {
errorMsg = "Failed to allocate videosource";
if (rv == NS_ERROR_NOT_AVAILABLE && !badConstraint) {
nsTArray<RefPtr<MediaDevice>> devices;
devices.AppendElement(mVideoDevice);
badConstraint = MediaConstraintsHelper::SelectSettings(
NormalizedConstraints(constraints), devices, mIsChrome);
}
if (mAudioDevice) {
mAudioDevice->Deallocate();
}
} else {
if (!mIsChrome) {
if (mShouldFocusSource) {
rv = mVideoDevice->FocusOnSelectedSource();
if (NS_FAILED(rv)) {
LOG("FocusOnSelectedSource failed");
}
}
}
}
}
if (errorMsg) {
LOG("%s %" PRIu32, errorMsg, static_cast<uint32_t>(rv));
if (badConstraint) {
Fail(MediaMgrError::Name::OverconstrainedError, u""_ns,
NS_ConvertUTF8toUTF16(badConstraint));
} else {
Fail(MediaMgrError::Name::NotReadableError,
NS_ConvertUTF8toUTF16(errorMsg));
}
NS_DispatchToMainThread(
NS_NewRunnableFunction("MediaManager::SendPendingGUMRequest", []() {
MediaManager* manager = MediaManager::GetIfExists();
if (!manager) {
return;
}
manager->SendPendingGUMRequest();
}));
return NS_OK;
}
PeerIdentity* peerIdentity = nullptr;
if (!mConstraints.mPeerIdentity.IsEmpty()) {
peerIdentity = new PeerIdentity(mConstraints.mPeerIdentity);
}
NS_DispatchToMainThread(do_AddRef(new GetUserMediaStreamRunnable(
std::move(mHolder), mWindowID, mWindowListener, mSourceListener,
mPrincipalInfo, mConstraints, mAudioDevice, mVideoDevice, peerIdentity,
mIsChrome)));
return NS_OK;
}
nsresult Denied(MediaMgrError::Name aName,
const nsString& aMessage = u""_ns) {
// We add a disabled listener to the StreamListeners array until accepted
// If this was the only active MediaStream, remove the window from the list.
if (NS_IsMainThread()) {
mHolder.Reject(MakeRefPtr<MediaMgrError>(aName, aMessage), __func__);
// Should happen *after* error runs for consistency, but may not matter
mWindowListener->Remove(mSourceListener);
} else {
// This will re-check the window being alive on main-thread
Fail(aName, aMessage);
}
return NS_OK;
}
nsresult SetContraints(const MediaStreamConstraints& aConstraints) {
mConstraints = aConstraints;
return NS_OK;
}
const MediaStreamConstraints& GetConstraints() { return mConstraints; }
nsresult SetAudioDevice(RefPtr<MediaDevice> aAudioDevice) {
mAudioDevice = std::move(aAudioDevice);
mDeviceChosen = true;
return NS_OK;
}
nsresult SetVideoDevice(RefPtr<MediaDevice> aVideoDevice) {
mVideoDevice = std::move(aVideoDevice);
mDeviceChosen = true;
return NS_OK;
}
uint64_t GetWindowID() { return mWindowID; }
private:
MediaStreamConstraints mConstraints;
MozPromiseHolder<MediaManager::StreamPromise> mHolder;
uint64_t mWindowID;
RefPtr<GetUserMediaWindowListener> mWindowListener;
RefPtr<SourceListener> mSourceListener;
RefPtr<MediaDevice> mAudioDevice;
RefPtr<MediaDevice> mVideoDevice;
const MediaEnginePrefs mPrefs;
ipc::PrincipalInfo mPrincipalInfo;
bool mIsChrome;
bool mShouldFocusSource;
bool mDeviceChosen;
public:
RefPtr<MediaManager::MediaDeviceSetRefCnt> mMediaDeviceSet;
private:
RefPtr<MediaManager> mManager; // get ref to this when creating the runnable
};
/* static */
void MediaManager::GuessVideoDeviceGroupIDs(MediaDeviceSet& aDevices,
const MediaDeviceSet& aAudios) {
// Run the logic in a lambda to avoid duplication.
auto updateGroupIdIfNeeded = [&](RefPtr<MediaDevice>& aVideo,
const MediaDeviceKind aKind) -> bool {
MOZ_ASSERT(aVideo->mKind == MediaDeviceKind::Videoinput);
MOZ_ASSERT(aKind == MediaDeviceKind::Audioinput ||
aKind == MediaDeviceKind::Audiooutput);
// This will store the new group id if a match is found.
nsString newVideoGroupID;
// If the group id needs to be updated this will become true. It is
// necessary when the new group id is an empty string. Without this extra
// variable to signal the update, we would resort to test if
// `newVideoGroupId` is empty. However,
// that check does not work when the new group id is an empty string.
bool updateGroupId = false;
for (const RefPtr<MediaDevice>& dev : aAudios) {
if (dev->mKind != aKind) {
continue;
}
if (!FindInReadable(aVideo->mName, dev->mName)) {
continue;
}
if (newVideoGroupID.IsEmpty()) {
// This is only expected on first match. If that's the only match group
// id will be updated to this one at the end of the loop.
updateGroupId = true;
newVideoGroupID = dev->mGroupID;
} else {
// More than one device found, it is impossible to know which group id
// is the correct one.
updateGroupId = false;
newVideoGroupID = u""_ns;
break;
}
}
if (updateGroupId) {
aVideo = new MediaDevice(aVideo, aVideo->mID, newVideoGroupID,
aVideo->mRawID, aVideo->mRawGroupID);
return true;
}
return false;
};
for (RefPtr<MediaDevice>& video : aDevices) {
if (video->mKind != MediaDeviceKind::Videoinput) {
continue;
}
if (updateGroupIdIfNeeded(video, MediaDeviceKind::Audioinput)) {
// GroupId has been updated, continue to the next video device
continue;
}
// GroupId has not been updated, check among the outputs
updateGroupIdIfNeeded(video, MediaDeviceKind::Audiooutput);
}
}
/**
* EnumerateRawDevices - Enumerate a list of audio & video devices that
* satisfy passed-in constraints. List contains raw id's.
*/
RefPtr<MediaManager::MgrPromise> MediaManager::EnumerateRawDevices(
uint64_t aWindowId, MediaSourceEnum aVideoInputType,
MediaSourceEnum aAudioInputType, MediaSinkEnum aAudioOutputType,
DeviceEnumerationType aVideoInputEnumType,
DeviceEnumerationType aAudioInputEnumType, bool aForceNoPermRequest,
const RefPtr<MediaDeviceSetRefCnt>& aOutDevices) {
MOZ_ASSERT(NS_IsMainThread());
MOZ_ASSERT(aVideoInputType != MediaSourceEnum::Other ||
aAudioInputType != MediaSourceEnum::Other ||
aAudioOutputType != MediaSinkEnum::Other);
// Since the enums can take one of several values, the following asserts rely
// on short circuting behavior. E.g. aVideoInputEnumType != Fake will be true
// if the requested device is not fake and thus the assert will pass. However,
// if the device is fake, aVideoInputType == MediaSourceEnum::Camera will be
// checked as well, ensuring that fake devices are of the camera type.
MOZ_ASSERT(aVideoInputEnumType != DeviceEnumerationType::Fake ||
aVideoInputType == MediaSourceEnum::Camera,
"If fake cams are requested video type should be camera!");
MOZ_ASSERT(aVideoInputEnumType != DeviceEnumerationType::Loopback ||
aVideoInputType == MediaSourceEnum::Camera,
"If loopback video is requested video type should be camera!");
MOZ_ASSERT(aAudioInputEnumType != DeviceEnumerationType::Fake ||
aAudioInputType == MediaSourceEnum::Microphone,
"If fake mics are requested audio type should be microphone!");
MOZ_ASSERT(aAudioInputEnumType != DeviceEnumerationType::Loopback ||
aAudioInputType == MediaSourceEnum::Microphone,
"If loopback audio is requested audio type should be microphone!");
LOG("%s: aWindowId=%" PRIu64 ", aVideoInputType=%" PRIu8
", aAudioInputType=%" PRIu8 ", aVideoInputEnumType=%" PRIu8
", aAudioInputEnumType=%" PRIu8,
__func__, aWindowId, static_cast<uint8_t>(aVideoInputType),
static_cast<uint8_t>(aAudioInputType),
static_cast<uint8_t>(aVideoInputEnumType),
static_cast<uint8_t>(aAudioInputEnumType));
auto holder = MakeUnique<MozPromiseHolder<MgrPromise>>();
RefPtr<MgrPromise> promise = holder->Ensure(__func__);
bool hasVideo = aVideoInputType != MediaSourceEnum::Other;
bool hasAudio = aAudioInputType != MediaSourceEnum::Other;
bool hasAudioOutput = aAudioOutputType == MediaSinkEnum::Speaker;
// True of at least one of video input or audio input is a fake device
bool fakeDeviceRequested =
(aVideoInputEnumType == DeviceEnumerationType::Fake && hasVideo) ||
(aAudioInputEnumType == DeviceEnumerationType::Fake && hasAudio);
// True if at least one of video input or audio input is a real device
// or there is audio output.
bool realDeviceRequested =
(aVideoInputEnumType != DeviceEnumerationType::Fake && hasVideo) ||
(aAudioInputEnumType != DeviceEnumerationType::Fake && hasAudio) ||
hasAudioOutput;
nsAutoCString videoLoopDev, audioLoopDev;
if (hasVideo && aVideoInputEnumType == DeviceEnumerationType::Loopback) {
Preferences::GetCString("media.video_loopback_dev", videoLoopDev);
}
if (hasAudio && aAudioInputEnumType == DeviceEnumerationType::Loopback) {
Preferences::GetCString("media.audio_loopback_dev", audioLoopDev);
}
RefPtr<Runnable> task = NewTaskFrom([holder = std::move(holder), aWindowId,
aVideoInputType, aAudioInputType,
aVideoInputEnumType, aAudioInputEnumType,
videoLoopDev, audioLoopDev, hasVideo,
hasAudio, hasAudioOutput,
fakeDeviceRequested, realDeviceRequested,
aOutDevices]() {
// Only enumerate what's asked for, and only fake cams and mics.
RefPtr<MediaEngine> fakeBackend, realBackend;
if (fakeDeviceRequested) {
fakeBackend = new MediaEngineDefault();
}
if (realDeviceRequested) {
MediaManager* manager = MediaManager::GetIfExists();
MOZ_RELEASE_ASSERT(manager); // Must exist while media thread is alive
realBackend = manager->GetBackend();
}
RefPtr<MediaEngine> videoBackend;
RefPtr<MediaEngine> audioBackend;
Maybe<MediaDeviceSet> micsOfVideoBackend;
Maybe<MediaDeviceSet> speakers;
if (hasVideo) {
videoBackend = aVideoInputEnumType == DeviceEnumerationType::Fake
? fakeBackend
: realBackend;
MediaDeviceSet videos;
LOG("EnumerateRawDevices Task: Getting video sources with %s backend",
videoBackend == fakeBackend ? "fake" : "real");
GetMediaDevices(videoBackend, aWindowId, aVideoInputType, videos,
videoLoopDev.get());
aOutDevices->AppendElements(videos);
}
if (hasAudio) {
audioBackend = aAudioInputEnumType == DeviceEnumerationType::Fake
? fakeBackend
: realBackend;
MediaDeviceSet audios;
LOG("EnumerateRawDevices Task: Getting audio sources with %s backend",
audioBackend == fakeBackend ? "fake" : "real");
GetMediaDevices(audioBackend, aWindowId, aAudioInputType, audios,
audioLoopDev.get());
if (aAudioInputType == MediaSourceEnum::Microphone &&
audioBackend == videoBackend) {
micsOfVideoBackend = Some(MediaDeviceSet());
micsOfVideoBackend->AppendElements(audios);
}