Source code

Revision control

Copy as Markdown

Other Tools

/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
const lazy = {};
ChromeUtils.defineESModuleGetters(lazy, {
TOGGLE_POLICY_STRINGS:
});
import { WebVTT } from "resource://gre/modules/vtt.sys.mjs";
import { setTimeout, clearTimeout } from "resource://gre/modules/Timer.sys.mjs";
import { XPCOMUtils } from "resource://gre/modules/XPCOMUtils.sys.mjs";
XPCOMUtils.defineLazyPreferenceGetter(
lazy,
"DISPLAY_TEXT_TRACKS_PREF",
"media.videocontrols.picture-in-picture.display-text-tracks.enabled",
false
);
XPCOMUtils.defineLazyPreferenceGetter(
lazy,
"IMPROVED_CONTROLS_ENABLED_PREF",
"media.videocontrols.picture-in-picture.improved-video-controls.enabled",
false
);
XPCOMUtils.defineLazyPreferenceGetter(
lazy,
"MIN_VIDEO_LENGTH",
"media.videocontrols.picture-in-picture.video-toggle.min-video-secs",
45
);
XPCOMUtils.defineLazyPreferenceGetter(
lazy,
"PIP_TOGGLE_ALWAYS_SHOW",
"media.videocontrols.picture-in-picture.video-toggle.always-show",
false
);
XPCOMUtils.defineLazyPreferenceGetter(
lazy,
"PIP_URLBAR_BUTTON",
"media.videocontrols.picture-in-picture.urlbar-button.enabled",
false
);
const PIP_ENABLED_PREF = "media.videocontrols.picture-in-picture.enabled";
const TOGGLE_ENABLED_PREF =
"media.videocontrols.picture-in-picture.video-toggle.enabled";
const TOGGLE_FIRST_SEEN_PREF =
"media.videocontrols.picture-in-picture.video-toggle.first-seen-secs";
const TOGGLE_FIRST_TIME_DURATION_DAYS = 28;
const TOGGLE_HAS_USED_PREF =
"media.videocontrols.picture-in-picture.video-toggle.has-used";
const TOGGLE_TESTING_PREF =
"media.videocontrols.picture-in-picture.video-toggle.testing";
const TOGGLE_VISIBILITY_THRESHOLD_PREF =
"media.videocontrols.picture-in-picture.video-toggle.visibility-threshold";
const TEXT_TRACK_FONT_SIZE =
"media.videocontrols.picture-in-picture.display-text-tracks.size";
const MOUSEMOVE_PROCESSING_DELAY_MS = 50;
const TOGGLE_HIDING_TIMEOUT_MS = 3000;
// If you change this, also change VideoControlsWidget.SEEK_TIME_SECS:
const SEEK_TIME_SECS = 5;
const EMPTIED_TIMEOUT_MS = 1000;
// The ToggleChild does not want to capture events from the PiP
// windows themselves. This set contains all currently open PiP
// players' content windows
var gPlayerContents = new WeakSet();
// To make it easier to write tests, we have a process-global
// WeakSet of all <video> elements that are being tracked for
// mouseover
var gWeakIntersectingVideosForTesting = new WeakSet();
// Overrides are expected to stay constant for the lifetime of a
// content process, so we set this as a lazy process global.
// See PictureInPictureToggleChild.getSiteOverrides for a
// sense of what the return types are.
ChromeUtils.defineLazyGetter(lazy, "gSiteOverrides", () => {
return PictureInPictureToggleChild.getSiteOverrides();
});
ChromeUtils.defineLazyGetter(lazy, "logConsole", () => {
return console.createInstance({
prefix: "PictureInPictureChild",
maxLogLevel: Services.prefs.getBoolPref(
"media.videocontrols.picture-in-picture.log",
false
)
? "Debug"
: "Error",
});
});
/**
* Creates and returns an instance of the PictureInPictureChildVideoWrapper class responsible
* for applying site-specific wrapper methods around the original video.
*
* The Picture-In-Picture add-on can use this to provide site-specific wrappers for
* sites that require special massaging to control.
* @param {Object} pipChild reference to PictureInPictureChild class calling this function
* @param {Element} originatingVideo
* The <video> element to wrap.
* @returns {PictureInPictureChildVideoWrapper} instance of PictureInPictureChildVideoWrapper
*/
function applyWrapper(pipChild, originatingVideo) {
let originatingDoc = originatingVideo.ownerDocument;
let originatingDocumentURI = originatingDoc.documentURI;
let overrides = lazy.gSiteOverrides.find(([matcher]) => {
return matcher.matches(originatingDocumentURI);
});
// gSiteOverrides is a list of tuples where the first element is the MatchPattern
// for a supported site and the second is the actual overrides object for it.
let wrapperPath = overrides ? overrides[1].videoWrapperScriptPath : null;
return new PictureInPictureChildVideoWrapper(
wrapperPath,
originatingVideo,
pipChild
);
}
export class PictureInPictureLauncherChild extends JSWindowActorChild {
handleEvent(event) {
switch (event.type) {
case "MozTogglePictureInPicture": {
if (event.isTrusted) {
this.togglePictureInPicture({
video: event.target,
reason: event.detail?.reason,
eventExtraKeys: event.detail?.eventExtraKeys,
});
}
break;
}
}
}
receiveMessage(message) {
switch (message.name) {
case "PictureInPicture:KeyToggle": {
this.keyToggle();
break;
}
}
}
/**
* Tells the parent to open a Picture-in-Picture window hosting
* a clone of the passed video. If we know about a pre-existing
* Picture-in-Picture window existing, this tells the parent to
* close it before opening the new one.
*
* @param {Object} pipObject
* @param {HTMLVideoElement} pipObject.video
* @param {String} pipObject.reason What toggled PiP, e.g. "shortcut"
* @param {Object} pipObject.eventExtraKeys Extra telemetry keys to record
*
* @return {Promise}
* @resolves {undefined} Once the new Picture-in-Picture window
* has been requested.
*/
async togglePictureInPicture(pipObject) {
let { video, reason, eventExtraKeys = {} } = pipObject;
if (video.isCloningElementVisually) {
// The only way we could have entered here for the same video is if
// we are toggling via the context menu or via the urlbar button,
// since we hide the inline Picture-in-Picture toggle when a video
// is being displayed in Picture-in-Picture. Turn off PiP in this case
const stopPipEvent = new this.contentWindow.CustomEvent(
"MozStopPictureInPicture",
{
bubbles: true,
detail: { reason },
}
);
video.dispatchEvent(stopPipEvent);
return;
}
if (!PictureInPictureChild.videoWrapper) {
PictureInPictureChild.videoWrapper = applyWrapper(
PictureInPictureChild,
video
);
}
let timestamp = undefined;
let scrubberPosition = undefined;
if (lazy.IMPROVED_CONTROLS_ENABLED_PREF) {
timestamp = PictureInPictureChild.videoWrapper.formatTimestamp(
PictureInPictureChild.videoWrapper.getCurrentTime(video),
PictureInPictureChild.videoWrapper.getDuration(video)
);
// Scrubber is hidden if undefined, so only set it to something else
// if the timestamp is not undefined.
scrubberPosition =
timestamp === undefined
? undefined
: PictureInPictureChild.videoWrapper.getCurrentTime(video) /
PictureInPictureChild.videoWrapper.getDuration(video);
}
// All other requests to toggle PiP should open a new PiP
// window
const videoRef = lazy.ContentDOMReference.get(video);
this.sendAsyncMessage("PictureInPicture:Request", {
isMuted: PictureInPictureChild.videoIsMuted(video),
playing: PictureInPictureChild.videoIsPlaying(video),
videoHeight: video.videoHeight,
videoWidth: video.videoWidth,
videoRef,
ccEnabled: lazy.DISPLAY_TEXT_TRACKS_PREF,
webVTTSubtitles: !!video.textTracks?.length,
scrubberPosition,
timestamp,
volume: PictureInPictureChild.videoWrapper.getVolume(video),
});
Services.telemetry.recordEvent(
"pictureinpicture",
"opened_method",
reason,
null,
{
firstTimeToggle: (!Services.prefs.getBoolPref(
TOGGLE_HAS_USED_PREF
)).toString(),
...eventExtraKeys,
}
);
}
/**
* The keyboard was used to attempt to open Picture-in-Picture. If a video is focused,
* select that video. Otherwise find the first playing video, or if none, the largest
* dimension video. We suspect this heuristic will handle most cases, though we
* might refine this later on. Note that we assume that this method will only be
* called for the focused document.
*/
keyToggle() {
let doc = this.document;
if (doc) {
let video = doc.activeElement;
if (!HTMLVideoElement.isInstance(video)) {
let listOfVideos = [...doc.querySelectorAll("video")].filter(
video => !isNaN(video.duration)
);
// Get the first non-paused video, otherwise the longest video. This
// fallback is designed to skip over "preview"-style videos on sidebars.
video =
listOfVideos.filter(v => !v.paused)[0] ||
listOfVideos.sort((a, b) => b.duration - a.duration)[0];
}
if (video) {
this.togglePictureInPicture({ video, reason: "shortcut" });
}
}
}
}
/**
* The PictureInPictureToggleChild is responsible for displaying the overlaid
* Picture-in-Picture toggle over top of <video> elements that the mouse is
* hovering.
*/
export class PictureInPictureToggleChild extends JSWindowActorChild {
constructor() {
super();
// We need to maintain some state about various things related to the
// Picture-in-Picture toggles - however, for now, the same
// PictureInPictureToggleChild might be re-used for different documents.
// We keep the state stashed inside of this WeakMap, keyed on the document
// itself.
this.weakDocStates = new WeakMap();
this.toggleEnabled =
Services.prefs.getBoolPref(TOGGLE_ENABLED_PREF) &&
Services.prefs.getBoolPref(PIP_ENABLED_PREF);
this.toggleTesting = Services.prefs.getBoolPref(TOGGLE_TESTING_PREF, false);
// Bug 1570744 - JSWindowActorChild's cannot be used as nsIObserver's
// directly, so we create a new function here instead to act as our
// nsIObserver, which forwards the notification to the observe method.
this.observerFunction = (subject, topic, data) => {
this.observe(subject, topic, data);
};
Services.prefs.addObserver(TOGGLE_ENABLED_PREF, this.observerFunction);
Services.prefs.addObserver(PIP_ENABLED_PREF, this.observerFunction);
Services.prefs.addObserver(TOGGLE_FIRST_SEEN_PREF, this.observerFunction);
Services.cpmm.sharedData.addEventListener("change", this);
this.eligiblePipVideos = new WeakSet();
this.trackingVideos = new WeakSet();
}
receiveMessage(message) {
switch (message.name) {
case "PictureInPicture:UrlbarToggle": {
this.urlbarToggle(message.data);
break;
}
}
return null;
}
didDestroy() {
this.stopTrackingMouseOverVideos();
Services.prefs.removeObserver(TOGGLE_ENABLED_PREF, this.observerFunction);
Services.prefs.removeObserver(PIP_ENABLED_PREF, this.observerFunction);
Services.prefs.removeObserver(
TOGGLE_FIRST_SEEN_PREF,
this.observerFunction
);
Services.cpmm.sharedData.removeEventListener("change", this);
// remove the observer on the <video> element
let state = this.docState;
if (state?.intersectionObserver) {
state.intersectionObserver.disconnect();
}
// ensure the sandbox created by the video is destroyed
this.videoWrapper?.destroy();
this.videoWrapper = null;
for (let video of ChromeUtils.nondeterministicGetWeakSetKeys(
this.eligiblePipVideos
)) {
video.removeEventListener("emptied", this);
video.removeEventListener("loadedmetadata", this);
video.removeEventListener("durationchange", this);
}
for (let video of ChromeUtils.nondeterministicGetWeakSetKeys(
this.trackingVideos
)) {
video.removeEventListener("emptied", this);
video.removeEventListener("loadedmetadata", this);
video.removeEventListener("durationchange", this);
}
// ensure we don't access the state
this.isDestroyed = true;
}
observe(subject, topic, data) {
if (topic != "nsPref:changed") {
return;
}
this.toggleEnabled =
Services.prefs.getBoolPref(TOGGLE_ENABLED_PREF) &&
Services.prefs.getBoolPref(PIP_ENABLED_PREF);
if (this.toggleEnabled) {
// We have enabled the Picture-in-Picture toggle, so we need to make
// sure we register all of the videos that might already be on the page.
this.contentWindow.requestIdleCallback(() => {
let videos = this.document.querySelectorAll("video");
for (let video of videos) {
this.registerVideo(video);
}
});
}
switch (data) {
case TOGGLE_FIRST_SEEN_PREF:
const firstSeenSeconds = Services.prefs.getIntPref(
TOGGLE_FIRST_SEEN_PREF
);
if (!firstSeenSeconds || firstSeenSeconds < 0) {
return;
}
this.changeToIconIfDurationEnd(firstSeenSeconds);
break;
}
}
/**
* Returns the state for the current document referred to via
* this.document. If no such state exists, creates it, stores it
* and returns it.
*/
get docState() {
if (this.isDestroyed || !this.document) {
return false;
}
let state = this.weakDocStates.get(this.document);
let visibilityThresholdPref = Services.prefs.getFloatPref(
TOGGLE_VISIBILITY_THRESHOLD_PREF,
"1.0"
);
if (!state) {
state = {
// A reference to the IntersectionObserver that's monitoring for videos
// to become visible.
intersectionObserver: null,
// A WeakSet of videos that are supposedly visible, according to the
// IntersectionObserver.
weakVisibleVideos: new WeakSet(),
// The number of videos that are supposedly visible, according to the
// IntersectionObserver
visibleVideosCount: 0,
// The DeferredTask that we'll arm every time a mousemove event occurs
// on a page where we have one or more visible videos.
mousemoveDeferredTask: null,
// A weak reference to the last video we displayed the toggle over.
weakOverVideo: null,
// True if the user is in the midst of clicking the toggle.
isClickingToggle: false,
// Set to the original target element on pointerdown if the user is clicking
// the toggle - this way, we can determine if a "click" event will need to be
// suppressed ("click" events don't fire if a "mouseup" occurs on a different
// element from the "pointerdown" / "mousedown" event).
clickedElement: null,
// This is a DeferredTask to hide the toggle after a period of mouse
// inactivity.
hideToggleDeferredTask: null,
// If we reach a point where we're tracking videos for mouse movements,
// then this will be true. If there are no videos worth tracking, then
// this is false.
isTrackingVideos: false,
togglePolicy: lazy.TOGGLE_POLICIES.DEFAULT,
toggleVisibilityThreshold: visibilityThresholdPref,
// The documentURI that has been checked with toggle policies and
// visibility thresholds for this document. Note that the documentURI
// might change for a document via the history API, so we remember
// the last checked documentURI to determine if we need to check again.
checkedPolicyDocumentURI: null,
};
this.weakDocStates.set(this.document, state);
}
return state;
}
/**
* Returns the video that the user was last hovering with the mouse if it
* still exists.
*
* @return {Element} the <video> element that the user was last hovering,
* or null if there was no such <video>, or the <video> no longer exists.
*/
getWeakOverVideo() {
let { weakOverVideo } = this.docState;
if (weakOverVideo) {
// Bug 800957 - Accessing weakrefs at the wrong time can cause us to
// throw NS_ERROR_XPC_BAD_CONVERT_NATIVE
try {
return weakOverVideo.get();
} catch (e) {
return null;
}
}
return null;
}
handleEvent(event) {
if (!event.isTrusted) {
// We don't care about synthesized events that might be coming from
// content JS.
return;
}
// Don't capture events from Picture-in-Picture content windows
if (gPlayerContents.has(this.contentWindow)) {
return;
}
switch (event.type) {
case "touchstart": {
// Even if this is a touch event, there may be subsequent click events.
// Suppress those events after selecting the toggle to prevent playback changes
// when opening the Picture-in-Picture window.
if (this.docState.isClickingToggle) {
event.stopImmediatePropagation();
event.preventDefault();
}
break;
}
case "change": {
const { changedKeys } = event;
if (changedKeys.includes("PictureInPicture:SiteOverrides")) {
// For now we only update our cache if the site overrides change.
// the user will need to refresh the page for changes to apply.
try {
lazy.gSiteOverrides =
PictureInPictureToggleChild.getSiteOverrides();
} catch (e) {
// Ignore resulting TypeError if gSiteOverrides is still unloaded
if (!(e instanceof TypeError)) {
throw e;
}
}
}
break;
}
case "UAWidgetSetupOrChange": {
if (
this.toggleEnabled &&
this.contentWindow.HTMLVideoElement.isInstance(event.target) &&
event.target.ownerDocument == this.document
) {
this.registerVideo(event.target);
}
break;
}
case "contextmenu": {
if (this.toggleEnabled) {
this.checkContextMenu(event);
}
break;
}
case "mouseout": {
this.onMouseOut(event);
break;
}
case "click":
if (event.detail == 0) {
let shadowRoot = event.originalTarget.containingShadowRoot;
let toggle = this.getToggleElement(shadowRoot);
if (event.originalTarget == toggle) {
this.startPictureInPicture(event, shadowRoot.host, toggle);
return;
}
}
// fall through
case "mousedown":
case "pointerup":
case "mouseup": {
this.onMouseButtonEvent(event);
break;
}
case "pointerdown": {
this.onPointerDown(event);
break;
}
case "mousemove": {
this.onMouseMove(event);
break;
}
case "pageshow": {
this.onPageShow(event);
break;
}
case "pagehide": {
this.onPageHide(event);
break;
}
case "durationchange":
// Intentional fall-through
case "emptied":
// Intentional fall-through
case "loadedmetadata": {
this.updatePipVideoEligibility(event.target);
break;
}
}
}
/**
* Adds a <video> to the IntersectionObserver so that we know when it becomes
* visible.
*
* @param {Element} video The <video> element to register.
*/
registerVideo(video) {
let state = this.docState;
if (!state.intersectionObserver) {
let fn = this.onIntersection.bind(this);
state.intersectionObserver = new this.contentWindow.IntersectionObserver(
fn,
{
threshold: [0.0, 0.5],
}
);
}
state.intersectionObserver.observe(video);
if (!lazy.PIP_URLBAR_BUTTON) {
return;
}
video.addEventListener("emptied", this);
video.addEventListener("loadedmetadata", this);
video.addEventListener("durationchange", this);
this.trackingVideos.add(video);
this.updatePipVideoEligibility(video);
}
updatePipVideoEligibility(video) {
let isEligible = this.isVideoPiPEligible(video);
if (isEligible) {
if (!this.eligiblePipVideos.has(video)) {
this.eligiblePipVideos.add(video);
let mutationObserver = new this.contentWindow.MutationObserver(
mutationList => {
this.handleEligiblePipVideoMutation(mutationList);
}
);
mutationObserver.observe(video.parentElement, { childList: true });
}
} else if (this.eligiblePipVideos.has(video)) {
this.eligiblePipVideos.delete(video);
}
let videos = ChromeUtils.nondeterministicGetWeakSetKeys(
this.eligiblePipVideos
);
this.sendAsyncMessage("PictureInPicture:UpdateEligiblePipVideoCount", {
pipCount: videos.length,
pipDisabledCount: videos.reduce(
(accumulator, currentVal) =>
accumulator + (currentVal.disablePictureInPicture ? 1 : 0),
0
),
});
}
handleEligiblePipVideoMutation(mutationList) {
for (let mutationRecord of mutationList) {
let video = mutationRecord.removedNodes[0];
this.eligiblePipVideos.delete(video);
}
let videos = ChromeUtils.nondeterministicGetWeakSetKeys(
this.eligiblePipVideos
);
this.sendAsyncMessage("PictureInPicture:UpdateEligiblePipVideoCount", {
pipCount: videos.length,
pipDisabledCount: videos.reduce(
(accumulator, currentVal) =>
accumulator + (currentVal.disablePictureInPicture ? 1 : 0),
0
),
});
}
urlbarToggle(eventExtraKeys) {
let video = ChromeUtils.nondeterministicGetWeakSetKeys(
this.eligiblePipVideos
)[0];
if (video) {
let pipEvent = new this.contentWindow.CustomEvent(
"MozTogglePictureInPicture",
{
bubbles: true,
detail: { reason: "urlBar", eventExtraKeys },
}
);
video.dispatchEvent(pipEvent);
}
}
isVideoPiPEligible(video) {
if (lazy.PIP_TOGGLE_ALWAYS_SHOW) {
return true;
}
if (isNaN(video.duration) || video.duration < lazy.MIN_VIDEO_LENGTH) {
return false;
}
const MIN_VIDEO_DIMENSION = 140; // pixels
if (
video.clientWidth < MIN_VIDEO_DIMENSION ||
video.clientHeight < MIN_VIDEO_DIMENSION
) {
return false;
}
return true;
}
/**
* Changes from the first-time toggle to the icon toggle if the Nimbus variable `displayDuration`'s
* end date is reached when hovering over a video. The end date is calculated according to the timestamp
* indicating when the PiP toggle was first seen.
* @param {Number} firstSeenStartSeconds the timestamp in seconds indicating when the PiP toggle was first seen
*/
changeToIconIfDurationEnd(firstSeenStartSeconds) {
const { displayDuration } =
lazy.NimbusFeatures.pictureinpicture.getAllVariables({
defaultValues: {
displayDuration: TOGGLE_FIRST_TIME_DURATION_DAYS,
},
});
if (!displayDuration || displayDuration < 0) {
return;
}
let daysInSeconds = displayDuration * 24 * 60 * 60;
let firstSeenEndSeconds = daysInSeconds + firstSeenStartSeconds;
let currentDateSeconds = Math.round(Date.now() / 1000);
lazy.logConsole.debug(
"Toggle duration experiment - first time toggle seen on:",
new Date(firstSeenStartSeconds * 1000).toLocaleDateString()
);
lazy.logConsole.debug(
"Toggle duration experiment - first time toggle will change on:",
new Date(firstSeenEndSeconds * 1000).toLocaleDateString()
);
lazy.logConsole.debug(
"Toggle duration experiment - current date:",
new Date(currentDateSeconds * 1000).toLocaleDateString()
);
if (currentDateSeconds >= firstSeenEndSeconds) {
this.sendAsyncMessage("PictureInPicture:SetHasUsed", {
hasUsed: true,
});
}
}
/**
* Called by the IntersectionObserver callback once a video becomes visible.
* This adds some fine-grained checking to ensure that a sufficient amount of
* the video is visible before we consider showing the toggles on it. For now,
* that means that the entirety of the video must be in the viewport.
*
* @param {IntersectionEntry} intersectionEntry An IntersectionEntry passed to
* the IntersectionObserver callback.
* @return bool Whether or not we should start tracking mousemove events for
* this registered video.
*/
worthTracking(intersectionEntry) {
return intersectionEntry.isIntersecting;
}
/**
* Called by the IntersectionObserver once a video crosses one of the
* thresholds dictated by the IntersectionObserver configuration.
*
* @param {Array<IntersectionEntry>} A collection of one or more
* IntersectionEntry's for <video> elements that might have entered or exited
* the viewport.
*/
onIntersection(entries) {
// The IntersectionObserver will also fire when a previously intersecting
// element is removed from the DOM. We know, however, that the node is
// still alive and referrable from the WeakSet because the
// IntersectionObserverEntry holds a strong reference to the video.
let state = this.docState;
if (!state) {
return;
}
let oldVisibleVideosCount = state.visibleVideosCount;
for (let entry of entries) {
let video = entry.target;
if (this.worthTracking(entry)) {
if (!state.weakVisibleVideos.has(video)) {
state.weakVisibleVideos.add(video);
state.visibleVideosCount++;
if (this.toggleTesting) {
gWeakIntersectingVideosForTesting.add(video);
}
}
} else if (state.weakVisibleVideos.has(video)) {
state.weakVisibleVideos.delete(video);
state.visibleVideosCount--;
if (this.toggleTesting) {
gWeakIntersectingVideosForTesting.delete(video);
}
}
}
// For testing, especially in debug or asan builds, we might not
// run this idle callback within an acceptable time. While we're
// testing, we'll bypass the idle callback performance optimization
// and run our callbacks as soon as possible during the next idle
// period.
if (!oldVisibleVideosCount && state.visibleVideosCount) {
if (this.toggleTesting || !this.contentWindow) {
this.beginTrackingMouseOverVideos();
} else {
this.contentWindow.requestIdleCallback(() => {
this.beginTrackingMouseOverVideos();
});
}
} else if (oldVisibleVideosCount && !state.visibleVideosCount) {
if (this.toggleTesting || !this.contentWindow) {
this.stopTrackingMouseOverVideos();
} else {
this.contentWindow.requestIdleCallback(() => {
this.stopTrackingMouseOverVideos();
});
}
}
}
addMouseButtonListeners() {
// We want to try to cancel the mouse events from continuing
// on into content if the user has clicked on the toggle, so
// we don't use the mozSystemGroup here, and add the listener
// to the parent target of the window, which in this case,
// is the windowRoot. Since this event listener is attached to
// part of the outer window, we need to also remove it in a
// pagehide event listener in the event that the page unloads
// before stopTrackingMouseOverVideos fires.
this.contentWindow.windowRoot.addEventListener("pointerdown", this, {
capture: true,
});
this.contentWindow.windowRoot.addEventListener("mousedown", this, {
capture: true,
});
this.contentWindow.windowRoot.addEventListener("mouseup", this, {
capture: true,
});
this.contentWindow.windowRoot.addEventListener("pointerup", this, {
capture: true,
});
this.contentWindow.windowRoot.addEventListener("click", this, {
capture: true,
});
this.contentWindow.windowRoot.addEventListener("mouseout", this, {
capture: true,
});
this.contentWindow.windowRoot.addEventListener("touchstart", this, {
capture: true,
});
}
removeMouseButtonListeners() {
// This can be null when closing the tab, but the event
// listeners should be removed in that case already.
if (!this.contentWindow || !this.contentWindow.windowRoot) {
return;
}
this.contentWindow.windowRoot.removeEventListener("pointerdown", this, {
capture: true,
});
this.contentWindow.windowRoot.removeEventListener("mousedown", this, {
capture: true,
});
this.contentWindow.windowRoot.removeEventListener("mouseup", this, {
capture: true,
});
this.contentWindow.windowRoot.removeEventListener("pointerup", this, {
capture: true,
});
this.contentWindow.windowRoot.removeEventListener("click", this, {
capture: true,
});
this.contentWindow.windowRoot.removeEventListener("mouseout", this, {
capture: true,
});
this.contentWindow.windowRoot.removeEventListener("touchstart", this, {
capture: true,
});
}
/**
* One of the challenges of displaying this toggle is that many sites put
* things over top of <video> elements, like custom controls, or images, or
* all manner of things that might intercept mouseevents that would normally
* fire directly on the <video>. In order to properly detect when the mouse
* is over top of one of the <video> elements in this situation, we currently
* add a mousemove event handler to the entire document, and stash the most
* recent mousemove that fires. At periodic intervals, that stashed mousemove
* event is checked to see if it's hovering over one of our registered
* <video> elements.
*
* This sort of thing will not be necessary once bug 1539652 is fixed.
*/
beginTrackingMouseOverVideos() {
let state = this.docState;
if (!state.mousemoveDeferredTask) {
state.mousemoveDeferredTask = new lazy.DeferredTask(() => {
this.checkLastMouseMove();
}, MOUSEMOVE_PROCESSING_DELAY_MS);
}
this.document.addEventListener("mousemove", this, {
mozSystemGroup: true,
capture: true,
});
this.contentWindow.addEventListener("pageshow", this, {
mozSystemGroup: true,
});
this.contentWindow.addEventListener("pagehide", this, {
mozSystemGroup: true,
});
this.addMouseButtonListeners();
state.isTrackingVideos = true;
}
/**
* If we no longer have any interesting videos in the viewport, we deregister
* the mousemove and click listeners, and also remove any toggles that might
* be on the page still.
*/
stopTrackingMouseOverVideos() {
let state = this.docState;
// We initialize `mousemoveDeferredTask` in `beginTrackingMouseOverVideos`.
// If it doesn't exist, that can't have happened. Nothing else ever sets
// this value (though we arm/disarm in various places). So we don't need
// to do anything else here and can return early.
if (!state.mousemoveDeferredTask) {
return;
}
state.mousemoveDeferredTask.disarm();
this.document.removeEventListener("mousemove", this, {
mozSystemGroup: true,
capture: true,
});
if (this.contentWindow) {
this.contentWindow.removeEventListener("pageshow", this, {
mozSystemGroup: true,
});
this.contentWindow.removeEventListener("pagehide", this, {
mozSystemGroup: true,
});
}
this.removeMouseButtonListeners();
let oldOverVideo = this.getWeakOverVideo();
if (oldOverVideo) {
this.onMouseLeaveVideo(oldOverVideo);
}
state.isTrackingVideos = false;
}
/**
* This pageshow event handler will get called if and when we complete a tab
* tear out or in. If we happened to be tracking videos before the tear
* occurred, we re-add the mouse event listeners so that they're attached to
* the right WindowRoot.
*
* @param {Event} event The pageshow event fired when completing a tab tear
* out or in.
*/
onPageShow(event) {
let state = this.docState;
if (state.isTrackingVideos) {
this.addMouseButtonListeners();
}
}
/**
* This pagehide event handler will get called if and when we start a tab
* tear out or in. If we happened to be tracking videos before the tear
* occurred, we remove the mouse event listeners. We'll re-add them when the
* pageshow event fires.
*
* @param {Event} event The pagehide event fired when starting a tab tear
* out or in.
*/
onPageHide(event) {
let state = this.docState;
if (state.isTrackingVideos) {
this.removeMouseButtonListeners();
}
}
/**
* If we're tracking <video> elements, this pointerdown event handler is run anytime
* a pointerdown occurs on the document. This function is responsible for checking
* if the user clicked on the Picture-in-Picture toggle. It does this by first
* checking if the video is visible beneath the point that was clicked. Then
* it tests whether or not the pointerdown occurred within the rectangle of the
* toggle. If so, the event's propagation is stopped, and Picture-in-Picture is
* triggered.
*
* @param {Event} event The mousemove event.
*/
onPointerDown(event) {
// The toggle ignores non-primary mouse clicks.
if (event.button != 0) {
return;
}
let video = this.getWeakOverVideo();
if (!video) {
return;
}
let shadowRoot = video.openOrClosedShadowRoot;
if (!shadowRoot) {
return;
}
let state = this.docState;
let overVideo = (() => {
let { clientX, clientY } = event;
let winUtils = this.contentWindow.windowUtils;
// We use winUtils.nodesFromRect instead of document.elementsFromPoint,
// since document.elementsFromPoint always flushes layout. The 1's in that
// function call are for the size of the rect that we want, which is 1x1.
//
// We pass the aOnlyVisible boolean argument to check that the video isn't
// occluded by anything visible at the point of mousedown. If it is, we'll
// ignore the mousedown.
let elements = winUtils.nodesFromRect(
clientX,
clientY,
1,
1,
1,
1,
true,
false,
/* aOnlyVisible = */ true,
state.toggleVisibilityThreshold
);
for (let element of elements) {
if (element == video || element.containingShadowRoot == shadowRoot) {
return true;
}
}
return false;
})();
if (!overVideo) {
return;
}
let toggle = this.getToggleElement(shadowRoot);
if (this.isMouseOverToggle(toggle, event)) {
state.isClickingToggle = true;
state.clickedElement = Cu.getWeakReference(event.originalTarget);
event.stopImmediatePropagation();
this.startPictureInPicture(event, video, toggle);
}
}
startPictureInPicture(event, video, toggle) {
Services.telemetry.keyedScalarAdd(
"pictureinpicture.opened_method",
"toggle",
1
);
let pipEvent = new this.contentWindow.CustomEvent(
"MozTogglePictureInPicture",
{
bubbles: true,
detail: { reason: "toggle" },
}
);
video.dispatchEvent(pipEvent);
// Since we've initiated Picture-in-Picture, we can go ahead and
// hide the toggle now.
this.onMouseLeaveVideo(video);
}
/**
* Called for mousedown, pointerup, mouseup and click events. If we
* detected that the user is clicking on the Picture-in-Picture toggle,
* these events are cancelled in the capture-phase before they reach
* content. The state for suppressing these events is cleared on the
* click event (unless the mouseup occurs on a different element from
* the mousedown, in which case, the state is cleared on mouseup).
*
* @param {Event} event A mousedown, pointerup, mouseup or click event.
*/
onMouseButtonEvent(event) {
// The toggle ignores non-primary mouse clicks.
if (event.button != 0) {
return;
}
let state = this.docState;
if (state.isClickingToggle) {
event.stopImmediatePropagation();
// If this is a mouseup event, check to see if we have a record of what
// the original target was on pointerdown. If so, and if it doesn't match
// the mouseup original target, that means we won't get a click event, and
// we can clear the "clicking the toggle" state right away.
//
// Otherwise, we wait for the click event to do that.
let isMouseUpOnOtherElement =
event.type == "mouseup" &&
(!state.clickedElement ||
state.clickedElement.get() != event.originalTarget);
if (
isMouseUpOnOtherElement ||
event.type == "click" ||
// pointerup event still triggers after a touchstart event. We just need to detect
// the pointer type and determine if we got to this part of the code through a touch event.
event.pointerType == "touch"
) {
// The click is complete, so now we reset the state so that
// we stop suppressing these events.
state.isClickingToggle = false;
state.clickedElement = null;
}
}
}
/**
* Called on mouseout events to determine whether or not the mouse has
* exited the window.
*
* @param {Event} event The mouseout event.
*/
onMouseOut(event) {
if (!event.relatedTarget) {
// For mouseout events, if there's no relatedTarget (which normally
// maps to the element that the mouse entered into) then this means that
// we left the window.
let video = this.getWeakOverVideo();
if (!video) {
return;
}
this.onMouseLeaveVideo(video);
}
}
/**
* Called for each mousemove event when we're tracking those events to
* determine if the cursor is hovering over a <video>.
*
* @param {Event} event The mousemove event.
*/
onMouseMove(event) {
let state = this.docState;
if (state.hideToggleDeferredTask) {
state.hideToggleDeferredTask.disarm();
state.hideToggleDeferredTask.arm();
}
state.lastMouseMoveEvent = event;
state.mousemoveDeferredTask.arm();
}
/**
* Called by the DeferredTask after MOUSEMOVE_PROCESSING_DELAY_MS
* milliseconds. Checked to see if that mousemove happens to be overtop of
* any interesting <video> elements that we want to display the toggle
* on. If so, puts the toggle on that video.
*/
checkLastMouseMove() {
let state = this.docState;
let event = state.lastMouseMoveEvent;
let { clientX, clientY } = event;
lazy.logConsole.debug("Visible videos count:", state.visibleVideosCount);
lazy.logConsole.debug("Tracking videos:", state.isTrackingVideos);
let winUtils = this.contentWindow.windowUtils;
// We use winUtils.nodesFromRect instead of document.elementsFromPoint,
// since document.elementsFromPoint always flushes layout. The 1's in that
// function call are for the size of the rect that we want, which is 1x1.
let elements = winUtils.nodesFromRect(
clientX,
clientY,
1,
1,
1,
1,
true,
false,
/* aOnlyVisible = */ true
);
for (let element of elements) {
lazy.logConsole.debug("Element id under cursor:", element.id);
lazy.logConsole.debug(
"Node name of an element under cursor:",
element.nodeName
);
lazy.logConsole.debug(
"Supported <video> element:",
state.weakVisibleVideos.has(element)
);
lazy.logConsole.debug(
"PiP window is open:",
element.isCloningElementVisually
);
// Check for hovering over the video controls or so too, not only
// directly over the video.
for (let el = element; el; el = el.containingShadowRoot?.host) {
if (state.weakVisibleVideos.has(el) && !el.isCloningElementVisually) {
lazy.logConsole.debug("Found supported element");
this.onMouseOverVideo(el, event);
return;
}
}
}
let oldOverVideo = this.getWeakOverVideo();
if (oldOverVideo) {
this.onMouseLeaveVideo(oldOverVideo);
}
}
/**
* Called once it has been determined that the mouse is overtop of a video
* that is in the viewport.
*
* @param {Element} video The video the mouse is over.
*/
onMouseOverVideo(video, event) {
let oldOverVideo = this.getWeakOverVideo();
let shadowRoot = video.openOrClosedShadowRoot;
if (shadowRoot.firstChild && video != oldOverVideo) {
if (video.getTransformToViewport().a == -1) {
shadowRoot.firstChild.setAttribute("flipped", true);
} else {
shadowRoot.firstChild.removeAttribute("flipped");
}
}
// It seems from automated testing that if it's still very early on in the
// lifecycle of a <video> element, it might not yet have a shadowRoot,
// in which case, we can bail out here early.
if (!shadowRoot) {
if (oldOverVideo) {
// We also clear the hover state on the old video we were hovering,
// if there was one.
this.onMouseLeaveVideo(oldOverVideo);
}
return;
}
let state = this.docState;
let toggle = this.getToggleElement(shadowRoot);
let controlsOverlay = shadowRoot.querySelector(".controlsOverlay");
if (state.checkedPolicyDocumentURI != this.document.documentURI) {
state.togglePolicy = lazy.TOGGLE_POLICIES.DEFAULT;
// We cache the matchers process-wide. We'll skip this while running tests to make that
// easier.
let siteOverrides = this.toggleTesting
? PictureInPictureToggleChild.getSiteOverrides()
: lazy.gSiteOverrides;
let visibilityThresholdPref = Services.prefs.getFloatPref(
TOGGLE_VISIBILITY_THRESHOLD_PREF,
"1.0"
);
if (!this.videoWrapper) {
this.videoWrapper = applyWrapper(this, video);
}
// Do we have any toggle overrides? If so, try to apply them.
for (let [override, { policy, visibilityThreshold }] of siteOverrides) {
if (
(policy || visibilityThreshold) &&
override.matches(this.document.documentURI)
) {
state.togglePolicy = this.videoWrapper?.shouldHideToggle(video)
? lazy.TOGGLE_POLICIES.HIDDEN
: policy || lazy.TOGGLE_POLICIES.DEFAULT;
state.toggleVisibilityThreshold =
visibilityThreshold || visibilityThresholdPref;
break;
}
}
state.checkedPolicyDocumentURI = this.document.documentURI;
}
// The built-in <video> controls are along the bottom, which would overlap the
// toggle if the override is set to BOTTOM, so we ignore overrides that set
// a policy of BOTTOM for <video> elements with controls.
if (
state.togglePolicy != lazy.TOGGLE_POLICIES.DEFAULT &&
!(state.togglePolicy == lazy.TOGGLE_POLICIES.BOTTOM && video.controls)
) {
toggle.setAttribute(
"policy",
lazy.TOGGLE_POLICY_STRINGS[state.togglePolicy]
);
} else {
toggle.removeAttribute("policy");
}
// nimbusExperimentVariables will be defaultValues when the experiment is disabled
const nimbusExperimentVariables =
lazy.NimbusFeatures.pictureinpicture.getAllVariables({
defaultValues: {
oldToggle: true,
title: null,
message: false,
showIconOnly: false,
displayDuration: TOGGLE_FIRST_TIME_DURATION_DAYS,
},
});
/**
* If a Nimbus variable exists for the first-time PiP toggle design,
* override the old design via a classname "experiment".
*/
if (!nimbusExperimentVariables.oldToggle) {
let controlsContainer = shadowRoot.querySelector(".controlsContainer");
let pipWrapper = shadowRoot.querySelector(".pip-wrapper");
controlsContainer.classList.add("experiment");
pipWrapper.classList.add("experiment");
} else {
let controlsContainer = shadowRoot.querySelector(".controlsContainer");
let pipWrapper = shadowRoot.querySelector(".pip-wrapper");
controlsContainer.classList.remove("experiment");
pipWrapper.classList.remove("experiment");
}
if (nimbusExperimentVariables.title) {
let pipExplainer = shadowRoot.querySelector(".pip-explainer");
let pipLabel = shadowRoot.querySelector(".pip-label");
if (pipExplainer && nimbusExperimentVariables.message) {
pipExplainer.innerText = nimbusExperimentVariables.message;
}
pipLabel.innerText = nimbusExperimentVariables.title;
} else if (nimbusExperimentVariables.showIconOnly) {
// We only want to show the PiP icon in this experiment scenario
let pipExpanded = shadowRoot.querySelector(".pip-expanded");
pipExpanded.style.display = "none";
let pipSmall = shadowRoot.querySelector(".pip-small");
pipSmall.style.opacity = "1";
let pipIcon = shadowRoot.querySelectorAll(".pip-icon")[1];
pipIcon.style.display = "block";
}
controlsOverlay.removeAttribute("hidetoggle");
// The hideToggleDeferredTask we create here is for automatically hiding
// the toggle after a period of no mousemove activity for
// TOGGLE_HIDING_TIMEOUT_MS. If the mouse moves, then the DeferredTask
// timer is reset.
//
// We disable the toggle hiding timeout during testing to reduce
// non-determinism from timers when testing the toggle.
if (!state.hideToggleDeferredTask && !this.toggleTesting) {
state.hideToggleDeferredTask = new lazy.DeferredTask(() => {
controlsOverlay.setAttribute("hidetoggle", true);
}, TOGGLE_HIDING_TIMEOUT_MS);
}
if (oldOverVideo) {
if (oldOverVideo == video) {
// If we're still hovering the old video, we might have entered or
// exited the toggle region.
this.checkHoverToggle(toggle, event);
return;
}
// We had an old video that we were hovering, and we're not hovering
// it anymore. Let's leave it.
this.onMouseLeaveVideo(oldOverVideo);
}
state.weakOverVideo = Cu.getWeakReference(video);
controlsOverlay.classList.add("hovering");
if (
state.togglePolicy != lazy.TOGGLE_POLICIES.HIDDEN &&
!toggle.hasAttribute("hidden")
) {
Services.telemetry.scalarAdd("pictureinpicture.saw_toggle", 1);
const hasUsedPiP = Services.prefs.getBoolPref(TOGGLE_HAS_USED_PREF);
let args = {
firstTime: (!hasUsedPiP).toString(),
};
Services.telemetry.recordEvent(
"pictureinpicture",
"saw_toggle",
"toggle",
null,
args
);
// only record if this is the first time seeing the toggle
if (!hasUsedPiP) {
lazy.NimbusFeatures.pictureinpicture.recordExposureEvent();
const firstSeenSeconds = Services.prefs.getIntPref(
TOGGLE_FIRST_SEEN_PREF,
0
);
if (!firstSeenSeconds || firstSeenSeconds < 0) {
let firstTimePiPStartDate = Math.round(Date.now() / 1000);
this.sendAsyncMessage("PictureInPicture:SetFirstSeen", {
dateSeconds: firstTimePiPStartDate,
});
} else if (nimbusExperimentVariables.displayDuration) {
this.changeToIconIfDurationEnd(firstSeenSeconds);
}
}
}
// Now that we're hovering the video, we'll check to see if we're
// hovering the toggle too.
this.checkHoverToggle(toggle, event);
}
/**
* Checks if a mouse event is happening over a toggle element. If it is,
* sets the hovering class on it. Otherwise, it clears the hovering
* class.
*
* @param {Element} toggle The Picture-in-Picture toggle to check.
* @param {MouseEvent} event A MouseEvent to test.
*/
checkHoverToggle(toggle, event) {
toggle.classList.toggle("hovering", this.isMouseOverToggle(toggle, event));
}
/**
* Called once it has been determined that the mouse is no longer overlapping
* a video that we'd previously called onMouseOverVideo with.
*
* @param {Element} video The video that the mouse left.
*/
onMouseLeaveVideo(video) {
let state = this.docState;
let shadowRoot = video.openOrClosedShadowRoot;
if (shadowRoot) {
let controlsOverlay = shadowRoot.querySelector(".controlsOverlay");
let toggle = this.getToggleElement(shadowRoot);
controlsOverlay.classList.remove("hovering");
toggle.classList.remove("hovering");
}
state.weakOverVideo = null;
if (!this.toggleTesting) {
state.hideToggleDeferredTask.disarm();
state.mousemoveDeferredTask.disarm();
}
state.hideToggleDeferredTask = null;
}
/**
* Given a reference to a Picture-in-Picture toggle element, determines
* if a MouseEvent event is occurring within its bounds.
*
* @param {Element} toggle The Picture-in-Picture toggle.
* @param {MouseEvent} event A MouseEvent to test.
*
* @return {Boolean}
*/
isMouseOverToggle(toggle, event) {
let toggleRect =
toggle.ownerGlobal.windowUtils.getBoundsWithoutFlushing(toggle);
// The way the toggle is currently implemented with
// absolute positioning, the root toggle element bounds don't actually
// contain all of the toggle child element bounds. Until we find a way to
// sort that out, we workaround the issue by having each clickable child
// elements of the toggle have a clicklable class, and then compute the
// smallest rect that contains all of their bounding rects and use that
// as the hitbox.
toggleRect = lazy.Rect.fromRect(toggleRect);
let clickableChildren = toggle.querySelectorAll(".clickable");
for (let child of clickableChildren) {
let childRect = lazy.Rect.fromRect(
child.ownerGlobal.windowUtils.getBoundsWithoutFlushing(child)
);
toggleRect.expandToContain(childRect);
}
// If the toggle has no dimensions, we're definitely not over it.
if (!toggleRect.width || !toggleRect.height) {
return false;
}
let { clientX, clientY } = event;
return (
clientX >= toggleRect.left &&
clientX <= toggleRect.right &&
clientY >= toggleRect.top &&
clientY <= toggleRect.bottom
);
}
/**
* Checks a contextmenu event to see if the mouse is currently over the
* Picture-in-Picture toggle. If so, sends a message to the parent process
* to open up the Picture-in-Picture toggle context menu.
*
* @param {MouseEvent} event A contextmenu event.
*/
checkContextMenu(event) {
let video = this.getWeakOverVideo();
if (!video) {
return;
}
let shadowRoot = video.openOrClosedShadowRoot;
if (!shadowRoot) {
return;
}
let toggle = this.getToggleElement(shadowRoot);
if (this.isMouseOverToggle(toggle, event)) {
let devicePixelRatio = toggle.ownerGlobal.devicePixelRatio;
this.sendAsyncMessage("PictureInPicture:OpenToggleContextMenu", {
screenXDevPx: event.screenX * devicePixelRatio,
screenYDevPx: event.screenY * devicePixelRatio,
inputSource: event.inputSource,
});
event.stopImmediatePropagation();
event.preventDefault();
}
}
/**
* Returns the appropriate root element for the Picture-in-Picture toggle,
* depending on whether or not we're using the experimental toggle preference.
*
* @param {Element} shadowRoot The shadowRoot of the video element.
* @returns {Element} The toggle element.
*/
getToggleElement(shadowRoot) {
return shadowRoot.getElementById("pictureInPictureToggle");
}
/**
* This is a test-only function that returns true if a video is being tracked
* for mouseover events after having intersected the viewport.
*/
static isTracking(video) {
return gWeakIntersectingVideosForTesting.has(video);
}
/**
* Gets any Picture-in-Picture site-specific overrides stored in the
* sharedData struct, and returns them as an Array of two-element Arrays,
* where the first element is a MatchPattern and the second element is an
* object of the form { policy, disabledKeyboardControls } (where each property
* may be missing or undefined).
*
* @returns {Array<Array<2>>} Array of 2-element Arrays where the first element
* is a MatchPattern and the second element is an object with optional policy
* and/or disabledKeyboardControls properties.
*/
static getSiteOverrides() {
let result = [];
let patterns = Services.cpmm.sharedData.get(
"PictureInPicture:SiteOverrides"
);
for (let pattern in patterns) {
let matcher = new MatchPattern(pattern);
result.push([matcher, patterns[pattern]]);
}
return result;
}
}
export class PictureInPictureChild extends JSWindowActorChild {
#subtitlesEnabled = false;
// A weak reference to this PiP window's video element
weakVideo = null;
// A weak reference to this PiP window's content window
weakPlayerContent = null;
// A reference to current WebVTT track currently displayed on the content window
_currentWebVTTTrack = null;
observerFunction = null;
observe(subject, topic, data) {
if (topic != "nsPref:changed") {
return;
}
switch (data) {
case "media.videocontrols.picture-in-picture.display-text-tracks.enabled": {
const originatingVideo = this.getWeakVideo();
let isTextTrackPrefEnabled = Services.prefs.getBoolPref(
"media.videocontrols.picture-in-picture.display-text-tracks.enabled"
);
// Enable or disable text track support
if (isTextTrackPrefEnabled) {
this.setupTextTracks(originatingVideo);
} else {
this.removeTextTracks(originatingVideo);
}
break;
}
}
}
/**
* Creates a link element with a reference to the css stylesheet needed
* for text tracks responsive styling.
* @returns {Element} the link element containing text tracks stylesheet.
*/
createTextTracksStyleSheet() {
let headStyleElement = this.document.createElement("link");
headStyleElement.setAttribute("rel", "stylesheet");
headStyleElement.setAttribute(
"href",
);
headStyleElement.setAttribute("type", "text/css");
return headStyleElement;
}
/**
* Sets up Picture-in-Picture to support displaying text tracks from WebVTT
* or if WebVTT isn't supported we will register the caption change mutation observer if
* the site wrapper exists.
*
* If the originating video supports WebVTT, try to read the
* active track and cues. Display any active cues on the pip window