Source code

Revision control

Copy as Markdown

Other Tools

/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
var EXPORTED_SYMBOLS = ["NewTabUtils"];
const { XPCOMUtils } = ChromeUtils.importESModule(
);
// Android tests don't import these properly, so guard against that
let shortURL = {};
let searchShortcuts = {};
let didSuccessfulImport = false;
try {
shortURL = ChromeUtils.import("resource://activity-stream/lib/ShortURL.jsm");
searchShortcuts = ChromeUtils.import(
);
didSuccessfulImport = true;
} catch (e) {
// The test failed to import these files
}
const lazy = {};
ChromeUtils.defineESModuleGetters(lazy, {
});
ChromeUtils.defineModuleGetter(
lazy,
"PageThumbs",
);
ChromeUtils.defineModuleGetter(
lazy,
"BinarySearch",
);
ChromeUtils.defineModuleGetter(
lazy,
"pktApi",
);
ChromeUtils.defineModuleGetter(
lazy,
"Pocket",
);
let BrowserWindowTracker;
try {
BrowserWindowTracker = ChromeUtils.import(
).BrowserWindowTracker;
} catch (e) {
// BrowserWindowTracker is used to determine devicePixelRatio in
// _addFavicons. We fallback to the value 2 if we can't find a window,
// so it's safe to do nothing with this here.
}
XPCOMUtils.defineLazyGetter(lazy, "gCryptoHash", function() {
return Cc["@mozilla.org/security/hash;1"].createInstance(Ci.nsICryptoHash);
});
XPCOMUtils.defineLazyGetter(lazy, "gUnicodeConverter", function() {
let converter = Cc[
"@mozilla.org/intl/scriptableunicodeconverter"
].createInstance(Ci.nsIScriptableUnicodeConverter);
converter.charset = "utf8";
return converter;
});
// Boolean preferences that control newtab content
const PREF_NEWTAB_ENABLED = "browser.newtabpage.enabled";
// The maximum number of results PlacesProvider retrieves from history.
const HISTORY_RESULTS_LIMIT = 100;
// The maximum number of links Links.getLinks will return.
const LINKS_GET_LINKS_LIMIT = 100;
// The gather telemetry topic.
const TOPIC_GATHER_TELEMETRY = "gather-telemetry";
// Some default frecency threshold for Activity Stream requests
const ACTIVITY_STREAM_DEFAULT_FRECENCY = 150;
// Some default query limit for Activity Stream requests
const ACTIVITY_STREAM_DEFAULT_LIMIT = 12;
// Some default seconds ago for Activity Stream recent requests
const ACTIVITY_STREAM_DEFAULT_RECENT = 5 * 24 * 60 * 60;
// The fallback value for the width of smallFavicon in pixels.
// This value will be multiplied by the current window's devicePixelRatio.
// If devicePixelRatio cannot be found, it will be multiplied by 2.
const DEFAULT_SMALL_FAVICON_WIDTH = 16;
const POCKET_UPDATE_TIME = 24 * 60 * 60 * 1000; // 1 day
const POCKET_INACTIVE_TIME = 7 * 24 * 60 * 60 * 1000; // 1 week
const PREF_POCKET_LATEST_SINCE = "extensions.pocket.settings.latestSince";
/**
* Calculate the MD5 hash for a string.
* @param aValue
* The string to convert.
* @return The base64 representation of the MD5 hash.
*/
function toHash(aValue) {
let value = lazy.gUnicodeConverter.convertToByteArray(aValue);
lazy.gCryptoHash.init(lazy.gCryptoHash.MD5);
lazy.gCryptoHash.update(value, value.length);
return lazy.gCryptoHash.finish(true);
}
/**
* Singleton that provides storage functionality.
*/
XPCOMUtils.defineLazyGetter(lazy, "Storage", function() {
return new LinksStorage();
});
function LinksStorage() {
// Handle migration of data across versions.
try {
if (this._storedVersion < this._version) {
// This is either an upgrade, or version information is missing.
if (this._storedVersion < 1) {
// Version 1 moved data from DOM Storage to prefs. Since migrating from
// version 0 is no more supported, we just reportError a dataloss later.
throw new Error("Unsupported newTab storage version");
}
// Add further migration steps here.
} else {
// This is a downgrade. Since we cannot predict future, upgrades should
// be backwards compatible. We will set the version to the old value
// regardless, so, on next upgrade, the migration steps will run again.
// For this reason, they should also be able to run multiple times, even
// on top of an already up-to-date storage.
}
} catch (ex) {
// Something went wrong in the update process, we can't recover from here,
// so just clear the storage and start from scratch (dataloss!).
Cu.reportError(
"Unable to migrate the newTab storage to the current version. " +
"Restarting from scratch.\n" +
ex
);
this.clear();
}
// Set the version to the current one.
this._storedVersion = this._version;
}
LinksStorage.prototype = {
get _version() {
return 1;
},
get _prefs() {
return Object.freeze({
pinnedLinks: "browser.newtabpage.pinned",
blockedLinks: "browser.newtabpage.blocked",
});
},
get _storedVersion() {
if (this.__storedVersion === undefined) {
// When the pref is not set, the storage version is unknown, so either:
// - it's a new profile
// - it's a profile where versioning information got lost
// In this case we still run through all of the valid migrations,
// starting from 1, as if it was a downgrade. As previously stated the
// migrations should already support running on an updated store.
this.__storedVersion = Services.prefs.getIntPref(
"browser.newtabpage.storageVersion",
1
);
}
return this.__storedVersion;
},
set _storedVersion(aValue) {
Services.prefs.setIntPref("browser.newtabpage.storageVersion", aValue);
this.__storedVersion = aValue;
},
/**
* Gets the value for a given key from the storage.
* @param aKey The storage key (a string).
* @param aDefault A default value if the key doesn't exist.
* @return The value for the given key.
*/
get: function Storage_get(aKey, aDefault) {
let value;
try {
let prefValue = Services.prefs.getStringPref(this._prefs[aKey]);
value = JSON.parse(prefValue);
} catch (e) {}
return value || aDefault;
},
/**
* Sets the storage value for a given key.
* @param aKey The storage key (a string).
* @param aValue The value to set.
*/
set: function Storage_set(aKey, aValue) {
// Page titles may contain unicode, thus use complex values.
Services.prefs.setStringPref(this._prefs[aKey], JSON.stringify(aValue));
},
/**
* Removes the storage value for a given key.
* @param aKey The storage key (a string).
*/
remove: function Storage_remove(aKey) {
Services.prefs.clearUserPref(this._prefs[aKey]);
},
/**
* Clears the storage and removes all values.
*/
clear: function Storage_clear() {
for (let key in this._prefs) {
this.remove(key);
}
},
};
/**
* Singleton that serves as a registry for all open 'New Tab Page's.
*/
var AllPages = {
/**
* The array containing all active pages.
*/
_pages: [],
/**
* Cached value that tells whether the New Tab Page feature is enabled.
*/
_enabled: null,
/**
* Adds a page to the internal list of pages.
* @param aPage The page to register.
*/
register: function AllPages_register(aPage) {
this._pages.push(aPage);
this._addObserver();
},
/**
* Removes a page from the internal list of pages.
* @param aPage The page to unregister.
*/
unregister: function AllPages_unregister(aPage) {
let index = this._pages.indexOf(aPage);
if (index > -1) {
this._pages.splice(index, 1);
}
},
/**
* Returns whether the 'New Tab Page' is enabled.
*/
get enabled() {
if (this._enabled === null) {
this._enabled = Services.prefs.getBoolPref(PREF_NEWTAB_ENABLED);
}
return this._enabled;
},
/**
* Enables or disables the 'New Tab Page' feature.
*/
set enabled(aEnabled) {
if (this.enabled != aEnabled) {
Services.prefs.setBoolPref(PREF_NEWTAB_ENABLED, !!aEnabled);
}
},
/**
* Returns the number of registered New Tab Pages (i.e. the number of open
* about:newtab instances).
*/
get length() {
return this._pages.length;
},
/**
* Updates all currently active pages but the given one.
* @param aExceptPage The page to exclude from updating.
* @param aReason The reason for updating all pages.
*/
update(aExceptPage, aReason = "") {
for (let page of this._pages.slice()) {
if (aExceptPage != page) {
page.update(aReason);
}
}
},
/**
* Implements the nsIObserver interface to get notified when the preference
* value changes or when a new copy of a page thumbnail is available.
*/
observe: function AllPages_observe(aSubject, aTopic, aData) {
if (aTopic == "nsPref:changed") {
// Clear the cached value.
switch (aData) {
case PREF_NEWTAB_ENABLED:
this._enabled = null;
break;
}
}
// and all notifications get forwarded to each page.
this._pages.forEach(function(aPage) {
aPage.observe(aSubject, aTopic, aData);
}, this);
},
/**
* Adds a preference and new thumbnail observer and turns itself into a
* no-op after the first invokation.
*/
_addObserver: function AllPages_addObserver() {
Services.prefs.addObserver(PREF_NEWTAB_ENABLED, this, true);
Services.obs.addObserver(this, "page-thumbnail:create", true);
this._addObserver = function() {};
},
QueryInterface: ChromeUtils.generateQI([
"nsIObserver",
"nsISupportsWeakReference",
]),
};
/**
* Singleton that keeps track of all pinned links and their positions in the
* grid.
*/
var PinnedLinks = {
/**
* The cached list of pinned links.
*/
_links: null,
/**
* The array of pinned links.
*/
get links() {
if (!this._links) {
this._links = lazy.Storage.get("pinnedLinks", []);
}
return this._links;
},
/**
* Pins a link at the given position.
* @param aLink The link to pin.
* @param aIndex The grid index to pin the cell at.
* @return true if link changes, false otherwise
*/
pin: function PinnedLinks_pin(aLink, aIndex) {
// Clear the link's old position, if any.
this.unpin(aLink);
// change pinned link into a history link
let changed = this._makeHistoryLink(aLink);
this.links[aIndex] = aLink;
this.save();
return changed;
},
/**
* Unpins a given link.
* @param aLink The link to unpin.
*/
unpin: function PinnedLinks_unpin(aLink) {
let index = this._indexOfLink(aLink);
if (index == -1) {
return;
}
let links = this.links;
links[index] = null;
// trim trailing nulls
let i = links.length - 1;
while (i >= 0 && links[i] == null) {
i--;
}
links.splice(i + 1);
this.save();
},
/**
* Saves the current list of pinned links.
*/
save: function PinnedLinks_save() {
lazy.Storage.set("pinnedLinks", this.links);
},
/**
* Checks whether a given link is pinned.
* @params aLink The link to check.
* @return whether The link is pinned.
*/
isPinned: function PinnedLinks_isPinned(aLink) {
return this._indexOfLink(aLink) != -1;
},
/**
* Resets the links cache.
*/
resetCache: function PinnedLinks_resetCache() {
this._links = null;
},
/**
* Finds the index of a given link in the list of pinned links.
* @param aLink The link to find an index for.
* @return The link's index.
*/
_indexOfLink: function PinnedLinks_indexOfLink(aLink) {
for (let i = 0; i < this.links.length; i++) {
let link = this.links[i];
if (link && link.url == aLink.url) {
return i;
}
}
// The given link is unpinned.
return -1;
},
/**
* Transforms link into a "history" link
* @param aLink The link to change
* @return true if link changes, false otherwise
*/
_makeHistoryLink: function PinnedLinks_makeHistoryLink(aLink) {
if (!aLink.type || aLink.type == "history") {
return false;
}
aLink.type = "history";
return true;
},
/**
* Replaces existing link with another link.
* @param aUrl The url of existing link
* @param aLink The replacement link
*/
replace: function PinnedLinks_replace(aUrl, aLink) {
let index = this._indexOfLink({ url: aUrl });
if (index == -1) {
return;
}
this.links[index] = aLink;
this.save();
},
};
/**
* Singleton that keeps track of all blocked links in the grid.
*/
var BlockedLinks = {
/**
* A list of objects that are observing blocked link changes.
*/
_observers: [],
/**
* The cached list of blocked links.
*/
_links: null,
/**
* Registers an object that will be notified when the blocked links change.
*/
addObserver(aObserver) {
this._observers.push(aObserver);
},
/**
* Remove the observers.
*/
removeObservers() {
this._observers = [];
},
/**
* The list of blocked links.
*/
get links() {
if (!this._links) {
this._links = lazy.Storage.get("blockedLinks", {});
}
return this._links;
},
/**
* Blocks a given link. Adjusts siteMap accordingly, and notifies listeners.
* @param aLink The link to block.
*/
block: function BlockedLinks_block(aLink) {
this._callObservers("onLinkBlocked", aLink);
this.links[toHash(aLink.url)] = 1;
this.save();
// Make sure we unpin blocked links.
PinnedLinks.unpin(aLink);
},
/**
* Unblocks a given link. Adjusts siteMap accordingly, and notifies listeners.
* @param aLink The link to unblock.
*/
unblock: function BlockedLinks_unblock(aLink) {
if (this.isBlocked(aLink)) {
delete this.links[toHash(aLink.url)];
this.save();
this._callObservers("onLinkUnblocked", aLink);
}
},
/**
* Saves the current list of blocked links.
*/
save: function BlockedLinks_save() {
lazy.Storage.set("blockedLinks", this.links);
},
/**
* Returns whether a given link is blocked.
* @param aLink The link to check.
*/
isBlocked: function BlockedLinks_isBlocked(aLink) {
return toHash(aLink.url) in this.links;
},
/**
* Checks whether the list of blocked links is empty.
* @return Whether the list is empty.
*/
isEmpty: function BlockedLinks_isEmpty() {
return !Object.keys(this.links).length;
},
/**
* Resets the links cache.
*/
resetCache: function BlockedLinks_resetCache() {
this._links = null;
},
_callObservers(methodName, ...args) {
for (let obs of this._observers) {
if (typeof obs[methodName] == "function") {
try {
obs[methodName](...args);
} catch (err) {
Cu.reportError(err);
}
}
}
},
};
/**
* Singleton that serves as the default link provider for the grid. It queries
* the history to retrieve the most frequently visited sites.
*/
var PlacesProvider = {
/**
* Set this to change the maximum number of links the provider will provide.
*/
maxNumLinks: HISTORY_RESULTS_LIMIT,
/**
* Must be called before the provider is used.
*/
init: function PlacesProvider_init() {
this._placesObserver = new PlacesWeakCallbackWrapper(
this.handlePlacesEvents.bind(this)
);
PlacesObservers.addListener(
["page-visited", "page-title-changed", "pages-rank-changed"],
this._placesObserver
);
},
/**
* Gets the current set of links delivered by this provider.
* @param aCallback The function that the array of links is passed to.
*/
getLinks: function PlacesProvider_getLinks(aCallback) {
let options = lazy.PlacesUtils.history.getNewQueryOptions();
options.maxResults = this.maxNumLinks;
// Sort by frecency, descending.
options.sortingMode =
Ci.nsINavHistoryQueryOptions.SORT_BY_FRECENCY_DESCENDING;
let links = [];
let callback = {
handleResult(aResultSet) {
let row;
while ((row = aResultSet.getNextRow())) {
let url = row.getResultByIndex(1);
if (LinkChecker.checkLoadURI(url)) {
let title = row.getResultByIndex(2);
let frecency = row.getResultByIndex(12);
let lastVisitDate = row.getResultByIndex(5);
links.push({
url,
title,
frecency,
lastVisitDate,
type: "history",
});
}
}
},
handleError(aError) {
// Should we somehow handle this error?
aCallback([]);
},
handleCompletion(aReason) {
// The Places query breaks ties in frecency by place ID descending, but
// that's different from how Links.compareLinks breaks ties, because
// compareLinks doesn't have access to place IDs. It's very important
// that the initial list of links is sorted in the same order imposed by
// compareLinks, because Links uses compareLinks to perform binary
// searches on the list. So, ensure the list is so ordered.
let i = 1;
let outOfOrder = [];
while (i < links.length) {
if (Links.compareLinks(links[i - 1], links[i]) > 0) {
outOfOrder.push(links.splice(i, 1)[0]);
} else {
i++;
}
}
for (let link of outOfOrder) {
i = lazy.BinarySearch.insertionIndexOf(
Links.compareLinks,
links,
link
);
links.splice(i, 0, link);
}
aCallback(links);
},
};
// Execute the query.
let query = lazy.PlacesUtils.history.getNewQuery();
lazy.PlacesUtils.history.asyncExecuteLegacyQuery(query, options, callback);
},
/**
* Registers an object that will be notified when the provider's links change.
* @param aObserver An object with the following optional properties:
* * onLinkChanged: A function that's called when a single link
* changes. It's passed the provider and the link object. Only the
* link's `url` property is guaranteed to be present. If its `title`
* property is present, then its title has changed, and the
* property's value is the new title. If any sort properties are
* present, then its position within the provider's list of links may
* have changed, and the properties' values are the new sort-related
* values. Note that this link may not necessarily have been present
* in the lists returned from any previous calls to getLinks.
* * onManyLinksChanged: A function that's called when many links
* change at once. It's passed the provider. You should call
* getLinks to get the provider's new list of links.
*/
addObserver: function PlacesProvider_addObserver(aObserver) {
this._observers.push(aObserver);
},
_observers: [],
handlePlacesEvents(aEvents) {
for (let event of aEvents) {
switch (event.type) {
case "page-visited": {
if (event.visitCount == 1 && event.lastKnownTitle) {
this._callObservers("onLinkChanged", {
url: event.url,
title: event.lastKnownTitle,
});
}
break;
}
case "page-title-changed": {
this._callObservers("onLinkChanged", {
url: event.url,
title: event.title,
});
break;
}
case "pages-rank-changed": {
this._callObservers("onManyLinksChanged");
break;
}
}
}
},
_callObservers: function PlacesProvider__callObservers(aMethodName, aArg) {
for (let obs of this._observers) {
if (obs[aMethodName]) {
try {
obs[aMethodName](this, aArg);
} catch (err) {
Cu.reportError(err);
}
}
}
},
};
/**
* Queries history to retrieve the most frecent sites. Emits events when the
* history changes.
*/
var ActivityStreamProvider = {
THUMB_FAVICON_SIZE: 96,
/**
* Shared adjustment for selecting potentially blocked links.
*/
_adjustLimitForBlocked({ ignoreBlocked, numItems }) {
// Just use the usual number if blocked links won't be filtered out
if (ignoreBlocked) {
return numItems;
}
// Additionally select the number of blocked links in case they're removed
return Object.keys(BlockedLinks.links).length + numItems;
},
/**
* Shared sub-SELECT to get the guid of a bookmark of the current url while
* avoiding LEFT JOINs on moz_bookmarks. This avoids gettings tags. The guid
* could be one of multiple possible guids. Assumes `moz_places h` is in FROM.
*/
_commonBookmarkGuidSelect: `(
SELECT guid
FROM moz_bookmarks b
WHERE fk = h.id
AND type = :bookmarkType
AND (
SELECT id
FROM moz_bookmarks p
WHERE p.id = b.parent
AND p.parent <> :tagsFolderId
) NOTNULL
) AS bookmarkGuid`,
/**
* Shared WHERE expression filtering out undesired pages, e.g., hidden,
* unvisited, and non-http/s urls. Assumes moz_places is in FROM / JOIN.
*
* NB: SUBSTR(url) is used even without an index instead of url_hash because
* most desired pages will match http/s, so it will only run on the ~10s of
* rows matched. If url_hash were to be used, it should probably *not* be used
* by the query optimizer as we primarily want it optimized for the other
* conditions, e.g., most frecent first.
*/
_commonPlacesWhere: `
AND hidden = 0
AND last_visit_date > 0
AND (SUBSTR(url, 1, 6) == "https:"
OR SUBSTR(url, 1, 5) == "http:")
`,
/**
* Shared parameters for getting correct bookmarks and LIMITed queries.
*/
_getCommonParams(aOptions, aParams = {}) {
return Object.assign(
{
bookmarkType: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK,
limit: this._adjustLimitForBlocked(aOptions),
tagsFolderId: lazy.PlacesUtils.tagsFolderId,
},
aParams
);
},
/**
* Shared columns for Highlights related queries.
*/
_highlightsColumns: [
"bookmarkGuid",
"description",
"guid",
"preview_image_url",
"title",
"url",
],
/**
* Shared post-processing of Highlights links.
*/
_processHighlights(aLinks, aOptions, aType) {
// Filter out blocked if necessary
if (!aOptions.ignoreBlocked) {
aLinks = aLinks.filter(
link =>
!BlockedLinks.isBlocked(
link.pocket_id ? { url: link.open_url } : link
)
);
}
// Limit the results to the requested number and set a type corresponding to
// which query selected it
return aLinks.slice(0, aOptions.numItems).map(item =>
Object.assign(item, {
type: aType,
})
);
},
/**
* From an Array of links, if favicons are present, convert to data URIs
*
* @param {Array} aLinks
* an array containing objects with favicon data and mimeTypes
*
* @returns {Array} an array of links with favicons as data uri
*/
_faviconBytesToDataURI(aLinks) {
return aLinks.map(link => {
if (link.favicon) {
let encodedData = btoa(String.fromCharCode.apply(null, link.favicon));
link.favicon = `data:${link.mimeType};base64,${encodedData}`;
delete link.mimeType;
}
if (link.smallFavicon) {
let encodedData = btoa(
String.fromCharCode.apply(null, link.smallFavicon)
);
link.smallFavicon = `data:${link.smallFaviconMimeType};base64,${encodedData}`;
delete link.smallFaviconMimeType;
}
return link;
});
},
/**
* Get favicon data (and metadata) for a uri. Fetches both the largest favicon
* available, for Activity Stream; and a normal-sized favicon, for the Urlbar.
*
* @param {nsIURI} aUri Page to check for favicon data
* @param {number} preferredFaviconWidth
* The preferred width of the of the normal-sized favicon in pixels.
* @returns A promise of an object (possibly empty) containing the data.
*/
async _loadIcons(aUri, preferredFaviconWidth) {
let iconData = {};
// Fetch the largest icon available.
let faviconData;
try {
faviconData = await lazy.PlacesUtils.promiseFaviconData(
aUri,
this.THUMB_FAVICON_SIZE
);
Object.assign(iconData, {
favicon: faviconData.data,
faviconLength: faviconData.dataLen,
faviconRef: faviconData.uri.ref,
faviconSize: faviconData.size,
mimeType: faviconData.mimeType,
});
} catch (e) {
// Return early because fetching the largest favicon is the primary
// purpose of NewTabUtils.
return null;
}
// Also fetch a smaller icon.
try {
faviconData = await lazy.PlacesUtils.promiseFaviconData(
aUri,
preferredFaviconWidth
);
Object.assign(iconData, {
smallFavicon: faviconData.data,
smallFaviconLength: faviconData.dataLen,
smallFaviconRef: faviconData.uri.ref,
smallFaviconSize: faviconData.size,
smallFaviconMimeType: faviconData.mimeType,
});
} catch (e) {
// Do nothing with the error since we still have the large favicon fields.
}
return iconData;
},
/**
* Computes favicon data for each url in a set of links
*
* @param {Array} links
* an array containing objects without favicon data or mimeTypes yet
*
* @returns {Promise} Returns a promise with the array of links with the largest
* favicon available (as a byte array), mimeType, byte array
* length, and favicon size (width)
*/
_addFavicons(aLinks) {
let win;
if (BrowserWindowTracker) {
win = BrowserWindowTracker.getTopWindow();
}
// We fetch two copies of a page's favicon: the largest available, for
// Activity Stream; and a smaller size appropriate for the Urlbar.
const preferredFaviconWidth =
DEFAULT_SMALL_FAVICON_WIDTH * (win ? win.devicePixelRatio : 2);
// Each link in the array needs a favicon for it's page - so we fire off a
// promise for each link to compute the favicon data and attach it back to
// the original link object. We must wait until all favicons for the array
// of links are computed before returning
return Promise.all(
aLinks.map(
link =>
// eslint-disable-next-line no-async-promise-executor
new Promise(async resolve => {
// Never add favicon data for pocket items
if (link.type === "pocket") {
resolve(link);
return;
}
let iconData;
try {
let linkUri = Services.io.newURI(link.url);
iconData = await this._loadIcons(linkUri, preferredFaviconWidth);
// Switch the scheme to try again with the other
if (!iconData) {
linkUri = linkUri
.mutate()
.setScheme(linkUri.scheme === "https" ? "http" : "https")
.finalize();
iconData = await this._loadIcons(
linkUri,
preferredFaviconWidth
);
}
} catch (e) {
// We just won't put icon data on the link
}
// Add the icon data to the link if we have any
resolve(Object.assign(link, iconData));
})
)
);
},
/**
* Helper function which makes the call to the Pocket API to fetch the user's
* saved Pocket items.
*/
fetchSavedPocketItems(requestData) {
const latestSince =
Services.prefs.getStringPref(PREF_POCKET_LATEST_SINCE, 0) * 1000;
// Do not fetch Pocket items for users that have been inactive for too long, or are not logged in
if (
!lazy.pktApi.isUserLoggedIn() ||
Date.now() - latestSince > POCKET_INACTIVE_TIME
) {
return Promise.resolve(null);
}
return new Promise((resolve, reject) => {
lazy.pktApi.retrieve(requestData, {
success(data) {
resolve(data);
},
error(error) {
reject(error);
},
});
});
},
/**
* Get the most recently Pocket-ed items from a user's Pocket list. See:
*
* @param {Object} aOptions
* {int} numItems: The max number of pocket items to fetch
*/
async getRecentlyPocketed(aOptions) {
const pocketSecondsAgo =
Math.floor(Date.now() / 1000) - ACTIVITY_STREAM_DEFAULT_RECENT;
const requestData = {
detailType: "complete",
count: aOptions.numItems,
since: pocketSecondsAgo,
};
let data;
try {
data = await this.fetchSavedPocketItems(requestData);
if (!data) {
return [];
}
} catch (e) {
Cu.reportError(e);
return [];
}
/* Extract relevant parts needed to show this card as a highlight:
* url, preview image, title, description, and the unique item_id
* necessary for Pocket to identify the item
*/
let items = Object.values(data.list)
// status "0" means not archived or deleted
.filter(item => item.status === "0")
.map(item => ({
date_added: item.time_added * 1000,
description: item.excerpt,
preview_image_url: item.image && item.image.src,
title: item.resolved_title,
url: item.resolved_url,
pocket_id: item.item_id,
open_url: item.open_url,
}));
// Append the query param to let Pocket know this item came from highlights
for (let item of items) {
let url = new URL(item.open_url);
url.searchParams.append("src", "fx_new_tab");
item.open_url = url.href;
}
return this._processHighlights(items, aOptions, "pocket");
},
/**
* Get most-recently-created visited bookmarks for Activity Stream.
*
* @param {Object} aOptions
* {num} bookmarkSecondsAgo: Maximum age of added bookmark.
* {bool} ignoreBlocked: Do not filter out blocked links.
* {int} numItems: Maximum number of items to return.
*/
async getRecentBookmarks(aOptions) {
const options = Object.assign(
{
bookmarkSecondsAgo: ACTIVITY_STREAM_DEFAULT_RECENT,
ignoreBlocked: false,
numItems: ACTIVITY_STREAM_DEFAULT_LIMIT,
},
aOptions || {}
);
const sqlQuery = `
SELECT
b.guid AS bookmarkGuid,
description,
h.guid,
preview_image_url,
b.title,
b.dateAdded / 1000 AS date_added,
url
FROM moz_bookmarks b
JOIN moz_bookmarks p
ON p.id = b.parent
JOIN moz_places h
ON h.id = b.fk
WHERE b.dateAdded >= :dateAddedThreshold
AND b.title NOTNULL
AND b.type = :bookmarkType
AND p.parent <> :tagsFolderId
${this._commonPlacesWhere}
ORDER BY b.dateAdded DESC
LIMIT :limit
`;
return this._processHighlights(
await this.executePlacesQuery(sqlQuery, {
columns: [...this._highlightsColumns, "date_added"],
params: this._getCommonParams(options, {
dateAddedThreshold:
(Date.now() - options.bookmarkSecondsAgo * 1000) * 1000,
}),
}),
options,
"bookmark"
);
},
/**
* Get total count of all bookmarks.
* Note: this includes default bookmarks
*
* @return {int} The number bookmarks in the places DB.
*/
async getTotalBookmarksCount() {
let sqlQuery = `
SELECT count(*) FROM moz_bookmarks b
JOIN moz_bookmarks t ON t.id = b.parent
AND t.parent <> :tags_folder
WHERE b.type = :type_bookmark
`;
const result = await this.executePlacesQuery(sqlQuery, {
params: {
tags_folder: lazy.PlacesUtils.tagsFolderId,
type_bookmark: lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK,
},
});
return result[0][0];
},
/**
* Get most-recently-visited history with metadata for Activity Stream.
*
* @param {Object} aOptions
* {bool} ignoreBlocked: Do not filter out blocked links.
* {int} numItems: Maximum number of items to return.
*/
async getRecentHistory(aOptions) {
const options = Object.assign(
{
ignoreBlocked: false,
numItems: ACTIVITY_STREAM_DEFAULT_LIMIT,
},
aOptions || {}
);
const sqlQuery = `
SELECT
${this._commonBookmarkGuidSelect},
description,
guid,
preview_image_url,
title,
url
FROM moz_places h
WHERE description NOTNULL
AND preview_image_url NOTNULL
${this._commonPlacesWhere}
ORDER BY last_visit_date DESC
LIMIT :limit
`;
return this._processHighlights(
await this.executePlacesQuery(sqlQuery, {
columns: this._highlightsColumns,
params: this._getCommonParams(options),
}),
options,
"history"
);
},
/*
* Gets the top frecent sites for Activity Stream.
*
* @param {Object} aOptions
* {bool} ignoreBlocked: Do not filter out blocked links.
* {int} numItems: Maximum number of items to return.
* {int} topsiteFrecency: Minimum amount of frecency for a site.
* {bool} onePerDomain: Dedupe the resulting list.
* {bool} includeFavicon: Include favicons if available.
* {string} hideWithSearchParam: URLs that contain this search param will be
* excluded from the returned links. This value should be either undefined
* or a string with one of the following forms:
* - undefined: Fall back to the value of pref
* `browser.newtabpage.activity-stream.hideTopSitesWithSearchParam`
* - "" (empty) - Disable this feature
* - "key" - Search param named "key" with any or no value
* - "key=" - Search param named "key" with no value
* - "key=value" - Search param named "key" with value "value"
*
* @returns {Promise} Returns a promise with the array of links as payload.
*/
async getTopFrecentSites(aOptions) {
const options = Object.assign(
{
ignoreBlocked: false,
numItems: ACTIVITY_STREAM_DEFAULT_LIMIT,
topsiteFrecency: ACTIVITY_STREAM_DEFAULT_FRECENCY,
onePerDomain: true,
includeFavicon: true,
hideWithSearchParam: Services.prefs.getCharPref(
"browser.newtabpage.activity-stream.hideTopSitesWithSearchParam",
""
),
},
aOptions || {}
);
// Double the item count in case the host is deduped between with www or
// not-www (i.e., 2 hosts) and an extra buffer for multiple pages per host.
const origNumItems = options.numItems;
if (options.onePerDomain) {
options.numItems *= 2 * 10;
}
// Keep this query fast with frecency-indexed lookups (even with excess
// rows) and shift the more complex logic to post-processing afterwards
const sqlQuery = `
SELECT
${this._commonBookmarkGuidSelect},
frecency,
guid,
last_visit_date / 1000 AS lastVisitDate,
rev_host,
title,
url,
"history" as type
FROM moz_places h
WHERE frecency >= :frecencyThreshold
${this._commonPlacesWhere}
ORDER BY frecency DESC
LIMIT :limit
`;
let links = await this.executePlacesQuery(sqlQuery, {
columns: [
"bookmarkGuid",
"frecency",
"guid",
"lastVisitDate",
"title",
"url",
"type",
],
params: this._getCommonParams(options, {
frecencyThreshold: options.topsiteFrecency,
}),
});
// Determine if the other link is "better" (larger frecency, more recent,
// lexicographically earlier url)
function isOtherBetter(link, other) {
if (other.frecency === link.frecency) {
if (other.lastVisitDate === link.lastVisitDate) {
return other.url < link.url;
}
return other.lastVisitDate > link.lastVisitDate;
}
return other.frecency > link.frecency;
}
// Update a host Map with the better link
function setBetterLink(map, link, hostMatcher, combiner = () => {}) {
const host = hostMatcher(link.url)[1];
if (map.has(host)) {
const other = map.get(host);
if (isOtherBetter(link, other)) {
link = other;
}
combiner(link, other);
}
map.set(host, link);
}
// Convert all links that are supposed to be a seach shortcut to its canonical URL
if (
didSuccessfulImport &&
Services.prefs.getBoolPref(
`browser.newtabpage.activity-stream.${searchShortcuts.SEARCH_SHORTCUTS_EXPERIMENT}`
)
) {
links.forEach(link => {
let searchProvider = searchShortcuts.getSearchProvider(
shortURL.shortURL(link)
);
if (searchProvider) {
link.url = searchProvider.url;
}
});
}
// Remove links that contain the hide-with search param.
if (options.hideWithSearchParam) {
let [key, value] = options.hideWithSearchParam.split("=");
links = links.filter(link => {
try {
let { searchParams } = new URL(link.url);
return value === undefined
? !searchParams.has(key)
: !searchParams.getAll(key).includes(value);
} catch (error) {}
return true;
});
}
// Remove any blocked links.
if (!options.ignoreBlocked) {
links = links.filter(link => !BlockedLinks.isBlocked(link));
}
if (options.onePerDomain) {
// De-dup the links.
const exactHosts = new Map();
for (const link of links) {
// First we want to find the best link for an exact host
setBetterLink(exactHosts, link, url => url.match(/:\/\/([^\/]+)/));
}
// Clean up exact hosts to dedupe as non-www hosts
const hosts = new Map();
for (const link of exactHosts.values()) {
setBetterLink(
hosts,
link,
url => url.match(/:\/\/(?:www\.)?([^\/]+)/),
// Combine frecencies when deduping these links
(targetLink, otherLink) => {
targetLink.frecency = link.frecency + otherLink.frecency;
}
);
}
links = [...hosts.values()];
}
// Pick out the top links using the same comparer as before
links = links.sort(isOtherBetter).slice(0, origNumItems);
if (!options.includeFavicon) {
return links;
}
// Get the favicons as data URI for now (until we use the favicon protocol)
return this._faviconBytesToDataURI(await this._addFavicons(links));
},
/**
* Gets a specific bookmark given some info about it
*
* @param {Obj} aInfo
* An object with one and only one of the following properties:
* - url
* - guid
* - parentGuid and index
*/
async getBookmark(aInfo) {
let bookmark = await lazy.PlacesUtils.bookmarks.fetch(aInfo);
if (!bookmark) {
return null;
}
let result = {};
result.bookmarkGuid = bookmark.guid;
result.bookmarkTitle = bookmark.title;
result.lastModified = bookmark.lastModified.getTime();
result.url = bookmark.url.href;
return result;
},
/**
* Count the number of visited urls grouped by day
*/
getUserMonthlyActivity() {
let sqlQuery = `
SELECT count(*),
strftime('%Y-%m-%d', visit_date/1000000.0, 'unixepoch', 'localtime') as date_format
FROM moz_historyvisits
WHERE visit_date > 0
AND visit_date > strftime('%s','now','localtime','start of day','-27 days','utc') * 1000000
GROUP BY date_format
ORDER BY date_format ASC
`;
return this.executePlacesQuery(sqlQuery);
},
/**
* Executes arbitrary query against places database
*
* @param {String} aQuery
* SQL query to execute
* @param {Object} [optional] aOptions
* aOptions.columns - an array of column names. if supplied the return
* items will consists of objects keyed on column names. Otherwise
* array of raw values is returned in the select order
* aOptions.param - an object of SQL binding parameters
*
* @returns {Promise} Returns a promise with the array of retrieved items
*/
async executePlacesQuery(aQuery, aOptions = {}) {
let { columns, params } = aOptions;
let items = [];
let queryError = null;
let conn = await lazy.PlacesUtils.promiseDBConnection();
await conn.executeCached(aQuery, params, (aRow, aCancel) => {
try {
let item = null;
// if columns array is given construct an object
if (columns && Array.isArray(columns)) {
item = {};
columns.forEach(column => {
item[column] = aRow.getResultByName(column);
});
} else {
// if no columns - make an array of raw values
item = [];
for (let i = 0; i < aRow.numEntries; i++) {
item.push(aRow.getResultByIndex(i));
}
}
items.push(item);
} catch (e) {
queryError = e;
aCancel();
}
});
if (queryError) {
throw new Error(queryError);
}
return items;
},
};
/**
* A set of actions which influence what sites shown on the Activity Stream page
*/
var ActivityStreamLinks = {
_savedPocketStories: null,
_pocketLastUpdated: 0,
_pocketLastLatest: 0,
/**
* Block a url
*
* @param {Object} aLink
* The link which contains a URL to add to the block list
*/
blockURL(aLink) {
BlockedLinks.block(aLink);
// If we're blocking a pocket item, invalidate the cache too
if (aLink.pocket_id) {
this._savedPocketStories = null;
}
},
onLinkBlocked(aLink) {
Services.obs.notifyObservers(null, "newtab-linkBlocked", aLink.url);
},
/**
* Adds a bookmark and opens up the Bookmark Dialog to show feedback that
* the bookmarking action has been successful
*
* @param {Object} aData
* aData.url The url to bookmark
* aData.title The title of the page to bookmark
* @param {Window} aBrowserWindow
* The current browser chrome window
*
* @returns {Promise} Returns a promise set to an object representing the bookmark
*/
addBookmark(aData, aBrowserWindow) {
const { url, title } = aData;
return aBrowserWindow.PlacesCommandHook.bookmarkLink(url, title);
},
/**
* Removes a bookmark
*
* @param {String} aBookmarkGuid
* The bookmark guid associated with the bookmark to remove
*
* @returns {Promise} Returns a promise at completion.
*/
deleteBookmark(aBookmarkGuid) {
return lazy.PlacesUtils.bookmarks.remove(aBookmarkGuid);
},
/**
* Removes a history link and unpins the URL if previously pinned
*
* @param {String} aUrl
* The url to be removed from history
*
* @returns {Promise} Returns a promise set to true if link was removed
*/
deleteHistoryEntry(aUrl) {
const url = aUrl;
PinnedLinks.unpin({ url });
return lazy.PlacesUtils.history.remove(url);
},
/**
* Helper function which makes the call to the Pocket API to delete an item from
* a user's saved to Pocket feed. Also, invalidate the Pocket stories cache
*
* @param {Integer} aItemID
* The unique pocket ID used to find the item to be deleted
*
*@returns {Promise} Returns a promise at completion
*/
deletePocketEntry(aItemID) {
this._savedPocketStories = null;
return new Promise((success, error) =>
lazy.pktApi.deleteItem(aItemID, { success, error })
);
},
/**
* Helper function which makes the call to the Pocket API to archive an item from
* a user's saved to Pocket feed. Also, invalidate the Pocket stories cache
*
* @param {Integer} aItemID
* The unique pocket ID used to find the item to be archived
*
*@returns {Promise} Returns a promise at completion
*/
archivePocketEntry(aItemID) {
this._savedPocketStories = null;
return new Promise((success, error) =>
lazy.pktApi.archiveItem(aItemID, { success, error })
);
},
/**
* Helper function which makes the call to the Pocket API to save an item to
* a user's saved to Pocket feed if they are logged in. Also, invalidate the
* Pocket stories cache
*
* @param {String} aUrl
* The URL belonging to the story being saved
* @param {String} aTitle
* The title belonging to the story being saved
* @param {Browser} aBrowser
* The target browser to show the doorhanger in
*
*@returns {Promise} Returns a promise at completion
*/
addPocketEntry(aUrl, aTitle, aBrowser) {
// If the user is not logged in, show the panel to prompt them to log in
if (!lazy.pktApi.isUserLoggedIn()) {
lazy.Pocket.savePage(aBrowser, aUrl, aTitle);
return Promise.resolve(null);
}
// If the user is logged in, just save the link to Pocket and Activity Stream
// will update the page
this._savedPocketStories = null;
return new Promise((success, error) => {
lazy.pktApi.addLink(aUrl, {
title: aTitle,
success,
error,
});
});
},
/**
* Get the Highlights links to show on Activity Stream
*
* @param {Object} aOptions
* {bool} excludeBookmarks: Don't add bookmark items.
* {bool} excludeHistory: Don't add history items.
* {bool} excludePocket: Don't add Pocket items.
* {bool} withFavicons: Add favicon data: URIs, when possible.
* {int} numItems: Maximum number of (bookmark or history) items to return.
*
* @return {Promise} Returns a promise with the array of links as the payload
*/
async getHighlights(aOptions = {}) {
aOptions.numItems = aOptions.numItems || ACTIVITY_STREAM_DEFAULT_LIMIT;
const results = [];
// First get bookmarks if we want them
if (!aOptions.excludeBookmarks) {
results.push(
...(await ActivityStreamProvider.getRecentBookmarks(aOptions))
);
}
// Add the Pocket items if we need more and want them
if (aOptions.numItems - results.length > 0 && !aOptions.excludePocket) {
const latestSince = ~~Services.prefs.getStringPref(
PREF_POCKET_LATEST_SINCE,
0
);
// Invalidate the cache, get new stories, and update timestamps if:
// 1. we do not have saved to Pocket stories already cached OR
// 2. it has been too long since we last got Pocket stories OR
// 3. there has been a paged saved to pocket since we last got new stories
if (
!this._savedPocketStories ||
Date.now() - this._pocketLastUpdated > POCKET_UPDATE_TIME ||
this._pocketLastLatest < latestSince
) {
this._savedPocketStories = await ActivityStreamProvider.getRecentlyPocketed(
aOptions
);
this._pocketLastUpdated = Date.now();
this._pocketLastLatest = latestSince;
}
results.push(...this._savedPocketStories);
}
// Add in history if we need more and want them
if (aOptions.numItems - results.length > 0 && !aOptions.excludeHistory) {
// Use the same numItems as bookmarks above in case we remove duplicates
const history = await ActivityStreamProvider.getRecentHistory(aOptions);
// Only include a url once in the result preferring the bookmark
const bookmarkUrls = new Set(results.map(({ url }) => url));
for (const page of history) {
if (!bookmarkUrls.has(page.url)) {
results.push(page);
// Stop adding pages once we reach the desired maximum
if (results.length === aOptions.numItems) {
break;
}
}
}
}
if (aOptions.withFavicons) {
return ActivityStreamProvider._faviconBytesToDataURI(
await ActivityStreamProvider._addFavicons(results)
);
}
return results;
},
/**
* Get the top sites to show on Activity Stream
*
* @return {Promise} Returns a promise with the array of links as the payload
*/
async getTopSites(aOptions = {}) {
return ActivityStreamProvider.getTopFrecentSites(aOptions);
},
};
/**
* Singleton that provides access to all links contained in the grid (including
* the ones that don't fit on the grid). A link is a plain object that looks
* like this:
*
* {
* title: "Mozilla",
* frecency: 1337,
* lastVisitDate: 1394678824766431,
* }
*/
var Links = {
/**
* The maximum number of links returned by getLinks.
*/
maxNumLinks: LINKS_GET_LINKS_LIMIT,
/**
* A mapping from each provider to an object { sortedLinks, siteMap, linkMap }.
* sortedLinks is the cached, sorted array of links for the provider.
* siteMap is a mapping from base domains to URL count associated with the domain.
* The count does not include blocked URLs. siteMap is used to look up a
* user's top sites that can be targeted with a suggested tile.
* linkMap is a Map from link URLs to link objects.
*/
_providers: new Map(),
/**
* The properties of link objects used to sort them.
*/
_sortProperties: ["frecency", "lastVisitDate", "url"],
/**
* List of callbacks waiting for the cache to be populated.
*/