Source code
Revision control
Copy as Markdown
Other Tools
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
import {
actionCreators as ac,
actionTypes as at,
} from "resource://activity-stream/common/Actions.mjs";
import { TippyTopProvider } from "resource://activity-stream/lib/TippyTopProvider.sys.mjs";
import {
insertPinned,
TOP_SITES_MAX_SITES_PER_ROW,
} from "resource://activity-stream/common/Reducers.sys.mjs";
import { Dedupe } from "resource://activity-stream/common/Dedupe.sys.mjs";
import {
shortURL,
shortHostname,
} from "resource://activity-stream/lib/ShortURL.sys.mjs";
import { getDefaultOptions } from "resource://activity-stream/lib/ActivityStreamStorage.sys.mjs";
import {
CUSTOM_SEARCH_SHORTCUTS,
SEARCH_SHORTCUTS_EXPERIMENT,
SEARCH_SHORTCUTS_SEARCH_ENGINES_PREF,
SEARCH_SHORTCUTS_HAVE_PINNED_PREF,
checkHasSearchEngine,
getSearchProvider,
} from "resource://activity-stream/lib/SearchShortcuts.sys.mjs";
const lazy = {};
ChromeUtils.defineESModuleGetters(lazy, {
FilterAdult: "resource://activity-stream/lib/FilterAdult.sys.mjs",
LinksCache: "resource://activity-stream/lib/LinksCache.sys.mjs",
NewTabUtils: "resource://gre/modules/NewTabUtils.sys.mjs",
NimbusFeatures: "resource://nimbus/ExperimentAPI.sys.mjs",
PageThumbs: "resource://gre/modules/PageThumbs.sys.mjs",
PersistentCache: "resource://activity-stream/lib/PersistentCache.sys.mjs",
Region: "resource://gre/modules/Region.sys.mjs",
Sampling: "resource://gre/modules/components-utils/Sampling.sys.mjs",
Screenshots: "resource://activity-stream/lib/Screenshots.sys.mjs",
});
ChromeUtils.defineLazyGetter(lazy, "log", () => {
const { Logger } = ChromeUtils.importESModule(
"resource://messaging-system/lib/Logger.sys.mjs"
);
return new Logger("TopSitesFeed");
});
// `contextId` is a unique identifier used by Contextual Services
const CONTEXT_ID_PREF = "browser.contextual-services.contextId";
ChromeUtils.defineLazyGetter(lazy, "contextId", () => {
let _contextId = Services.prefs.getStringPref(CONTEXT_ID_PREF, null);
if (!_contextId) {
_contextId = String(Services.uuid.generateUUID());
Services.prefs.setStringPref(CONTEXT_ID_PREF, _contextId);
}
return _contextId;
});
const DEFAULT_SITES_PREF = "default.sites";
const SHOWN_ON_NEWTAB_PREF = "feeds.topsites";
export const DEFAULT_TOP_SITES = [];
const FRECENCY_THRESHOLD = 100 + 1; // 1 visit (skip first-run/one-time pages)
const MIN_FAVICON_SIZE = 96;
const CACHED_LINK_PROPS_TO_MIGRATE = ["screenshot", "customScreenshot"];
const PINNED_FAVICON_PROPS_TO_MIGRATE = [
"favicon",
"faviconRef",
"faviconSize",
];
const SECTION_ID = "topsites";
const CACHE_KEY = "contile";
const ROWS_PREF = "topSitesRows";
const SHOW_SPONSORED_PREF = "showSponsoredTopSites";
// The default total number of sponsored top sites to fetch from Contile
// and Pocket.
const MAX_NUM_SPONSORED = 3;
// Nimbus variable for the total number of sponsored top sites including
// both Contile and Pocket sources.
// The default will be `MAX_NUM_SPONSORED` if this variable is unspecified.
const NIMBUS_VARIABLE_MAX_SPONSORED = "topSitesMaxSponsored";
// Nimbus variable to allow more than two sponsored tiles from Contile to be
//considered for Top Sites.
const NIMBUS_VARIABLE_ADDITIONAL_TILES =
"topSitesUseAdditionalTilesFromContile";
// Nimbus variable to enable the SOV feature for sponsored tiles.
const NIMBUS_VARIABLE_CONTILE_SOV_ENABLED = "topSitesContileSovEnabled";
// Nimbu variable for the total number of sponsor topsite that come from Contile
// The default will be `CONTILE_MAX_NUM_SPONSORED` if variable is unspecified.
const NIMBUS_VARIABLE_CONTILE_MAX_NUM_SPONSORED = "topSitesContileMaxSponsored";
const PREF_UNIFIED_ADS_TILES_ENABLED = "unifiedAds.tiles.enabled";
const PREF_UNIFIED_ADS_ENDPOINT = "unifiedAds.endpoint";
const PREF_UNIFIED_ADS_PLACEMENTS = "discoverystream.placements.tiles";
const PREF_UNIFIED_ADS_COUNTS = "discoverystream.placements.tiles.counts";
const PREF_UNIFIED_ADS_BLOCKED_LIST = "unifiedAds.blockedAds";
// Search experiment stuff
const FILTER_DEFAULT_SEARCH_PREF = "improvesearch.noDefaultSearchTile";
const SEARCH_FILTERS = [
"google",
"search.yahoo",
"yahoo",
"bing",
"ask",
"duckduckgo",
];
const REMOTE_SETTING_DEFAULTS_PREF = "browser.topsites.useRemoteSetting";
const DEFAULT_SITES_OVERRIDE_PREF =
"browser.newtabpage.activity-stream.default.sites";
const DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH = "browser.topsites.experiment.";
// Mozilla Tiles Service (Contile) prefs
// Nimbus variable for the Contile integration. It falls back to the pref:
// `browser.topsites.contile.enabled`.
const NIMBUS_VARIABLE_CONTILE_ENABLED = "topSitesContileEnabled";
const NIMBUS_VARIABLE_CONTILE_POSITIONS = "contileTopsitesPositions";
const CONTILE_ENDPOINT_PREF = "browser.topsites.contile.endpoint";
const CONTILE_UPDATE_INTERVAL = 15 * 60 * 1000; // 15 minutes
// The maximum number of sponsored top sites to fetch from Contile.
const CONTILE_MAX_NUM_SPONSORED = 3;
const TOP_SITES_BLOCKED_SPONSORS_PREF = "browser.topsites.blockedSponsors";
const CONTILE_CACHE_VALID_FOR_PREF = "browser.topsites.contile.cacheValidFor";
const CONTILE_CACHE_LAST_FETCH_PREF = "browser.topsites.contile.lastFetch";
const CONTILE_CACHE_VALID_FOR_FALLBACK = 3 * 60 * 60; // 3 hours in seconds
// Partners of sponsored tiles.
const SPONSORED_TILE_PARTNER_AMP = "amp";
const SPONSORED_TILE_PARTNER_MOZ_SALES = "moz-sales";
const SPONSORED_TILE_PARTNERS = new Set([
SPONSORED_TILE_PARTNER_AMP,
SPONSORED_TILE_PARTNER_MOZ_SALES,
]);
const DISPLAY_FAIL_REASON_OVERSOLD = "oversold";
const DISPLAY_FAIL_REASON_DISMISSED = "dismissed";
const DISPLAY_FAIL_REASON_UNRESOLVED = "unresolved";
function getShortHostnameForCurrentSearch() {
return shortHostname(Services.search.defaultEngine.searchUrlDomain);
}
class TopSitesTelemetry {
constructor() {
this.allSponsoredTiles = {};
this.sponsoredTilesConfigured = 0;
}
_tileProviderForTiles(tiles) {
// Assumption: the list of tiles is from a single provider
return tiles && tiles.length ? this._tileProvider(tiles[0]) : null;
}
_tileProvider(tile) {
return tile.partner || SPONSORED_TILE_PARTNER_AMP;
}
_buildPropertyKey(tile) {
let provider = this._tileProvider(tile);
return provider + shortURL(tile);
}
// Returns an array of strings indicating the property name (based on the
// provider and brand) of tiles that have been filtered e.g. ["moz-salesbrand1"]
// currentTiles: The list of tiles remaining and may be displayed in new tab.
// this.allSponsoredTiles: The original list of tiles set via setTiles prior to any filtering
// The returned list indicated the difference between these two lists (excluding any previously filtered tiles).
_getFilteredTiles(currentTiles) {
let notPreviouslyFilteredTiles = Object.assign(
{},
...Object.entries(this.allSponsoredTiles)
.filter(
([, v]) =>
v.display_fail_reason === null ||
v.display_fail_reason === undefined
)
.map(([k, v]) => ({ [k]: v }))
);
// Get the property names of the newly filtered list.
let remainingTiles = currentTiles.map(el => {
return this._buildPropertyKey(el);
});
// Get the property names of the tiles that were filtered.
let tilesToUpdate = Object.keys(notPreviouslyFilteredTiles).filter(
element => !remainingTiles.includes(element)
);
return tilesToUpdate;
}
setSponsoredTilesConfigured() {
const maxSponsored =
lazy.NimbusFeatures.pocketNewtab.getVariable(
NIMBUS_VARIABLE_MAX_SPONSORED
) ?? MAX_NUM_SPONSORED;
this.sponsoredTilesConfigured = maxSponsored;
Glean.topsites.sponsoredTilesConfigured.set(maxSponsored);
}
clearTilesForProvider(provider) {
Object.entries(this.allSponsoredTiles)
.filter(([k]) => k.startsWith(provider))
.map(([k]) => delete this.allSponsoredTiles[k]);
}
_getAdvertiser(tile) {
let label = tile.label || null;
let title = tile.title || null;
return label ?? title ?? shortURL(tile);
}
setTiles(tiles) {
// Assumption: the list of tiles is from a single provider,
// should be called once per tile source.
if (tiles && tiles.length) {
let tile_provider = this._tileProviderForTiles(tiles);
this.clearTilesForProvider(tile_provider);
for (let sponsoredTile of tiles) {
this.allSponsoredTiles[this._buildPropertyKey(sponsoredTile)] = {
advertiser: this._getAdvertiser(sponsoredTile).toLowerCase(),
provider: tile_provider,
display_position: null,
display_fail_reason: null,
};
}
}
}
_setDisplayFailReason(filteredTiles, reason) {
for (let tile of filteredTiles) {
if (tile in this.allSponsoredTiles) {
let tileToUpdate = this.allSponsoredTiles[tile];
tileToUpdate.display_position = null;
tileToUpdate.display_fail_reason = reason;
}
}
}
determineFilteredTilesAndSetToOversold(nonOversoldTiles) {
let filteredTiles = this._getFilteredTiles(nonOversoldTiles);
this._setDisplayFailReason(filteredTiles, DISPLAY_FAIL_REASON_OVERSOLD);
}
determineFilteredTilesAndSetToDismissed(nonDismissedTiles) {
let filteredTiles = this._getFilteredTiles(nonDismissedTiles);
this._setDisplayFailReason(filteredTiles, DISPLAY_FAIL_REASON_DISMISSED);
}
_setTilePositions(currentTiles) {
// This function performs many loops over a small dataset. The size of
// dataset is limited by the number of sponsored tiles displayed on
// the newtab instance.
if (this.allSponsoredTiles) {
let tilePositionsAssigned = [];
// processing the currentTiles parameter, assigns a position to the
// corresponding property in this.allSponsoredTiles
currentTiles.forEach(item => {
let tile = this.allSponsoredTiles[this._buildPropertyKey(item)];
if (
tile &&
(tile.display_fail_reason === undefined ||
tile.display_fail_reason === null)
) {
tile.display_position = item.sponsored_position;
// Track assigned tile slots.
tilePositionsAssigned.push(item.sponsored_position);
}
});
// Need to check if any objects in this.allSponsoredTiles do not
// have either a display_fail_reason or a display_position set.
// This can happen if the tiles list was updated before the
// metric is written to Glean.
let tilesMissingPosition = [];
Object.keys(this.allSponsoredTiles).forEach(property => {
let tile = this.allSponsoredTiles[property];
if (!tile.display_fail_reason && !tile.display_position) {
tilesMissingPosition.push(property);
}
});
if (tilesMissingPosition.length) {
// Determine if any available slots exist based on max number of tiles
// and the list of tiles already used and assign to a tile with missing
// value.
for (let i = 1; i <= this.sponsoredTilesConfigured; i++) {
if (!tilePositionsAssigned.includes(i)) {
let tileProperty = tilesMissingPosition.shift();
this.allSponsoredTiles[tileProperty].display_position = i;
}
}
}
// At this point we might still have a few unresolved states. These
// rows will be tagged with a display_fail_reason `unresolved`.
this._detectErrorConditionAndSetUnresolved();
}
}
// Checks the data for inconsistent state and updates the display_fail_reason
_detectErrorConditionAndSetUnresolved() {
Object.keys(this.allSponsoredTiles).forEach(property => {
let tile = this.allSponsoredTiles[property];
if (
(!tile.display_fail_reason && !tile.display_position) ||
(tile.display_fail_reason && tile.display_position)
) {
tile.display_position = null;
tile.display_fail_reason = DISPLAY_FAIL_REASON_UNRESOLVED;
}
});
}
finalizeNewtabPingFields(currentTiles) {
this._setTilePositions(currentTiles);
Glean.topsites.sponsoredTilesReceived.set(
JSON.stringify({
sponsoredTilesReceived: Object.values(this.allSponsoredTiles),
})
);
}
}
export class ContileIntegration {
constructor(topSitesFeed) {
this._topSitesFeed = topSitesFeed;
this._lastPeriodicUpdate = 0;
this._sites = [];
// The Share-of-Voice object managed by Shepherd and sent via Contile.
this._sov = null;
this.cache = this.PersistentCache(CACHE_KEY, true);
}
get sites() {
return this._sites;
}
get sov() {
return this._sov;
}
periodicUpdate() {
let now = Date.now();
if (now - this._lastPeriodicUpdate >= CONTILE_UPDATE_INTERVAL) {
this._lastPeriodicUpdate = now;
this.refresh();
}
}
async refresh() {
let updateDefaultSites = await this._fetchSites();
await this._topSitesFeed.allocatePositions();
if (updateDefaultSites) {
this._topSitesFeed._readDefaults();
}
}
/**
* Clear Contile Cache.
*/
_resetContileCache() {
Services.prefs.clearUserPref(CONTILE_CACHE_LAST_FETCH_PREF);
Services.prefs.clearUserPref(CONTILE_CACHE_VALID_FOR_PREF);
// This can be async, but in this case we don't need to wait.
this.cache.set("contile", []);
}
/**
* Filter the tiles whose sponsor is on the Top Sites sponsor blocklist.
*
* @param {array} tiles
* An array of the tile objects
*/
_filterBlockedSponsors(tiles) {
const blocklist = JSON.parse(
Services.prefs.getStringPref(TOP_SITES_BLOCKED_SPONSORS_PREF, "[]")
);
return tiles.filter(tile => !blocklist.includes(shortURL(tile)));
}
/**
* Calculate the time Contile response is valid for based on cache-control header
*
* @param {string} cacheHeader
* string value of the Contile resposne cache-control header
*/
_extractCacheValidFor(cacheHeader) {
const unifiedAdsTilesEnabled =
this._topSitesFeed.store.getState().Prefs.values[
PREF_UNIFIED_ADS_TILES_ENABLED
];
// Note: Cache-control only applies to direct Contile API calls
if (!cacheHeader && !unifiedAdsTilesEnabled) {
lazy.log.warn("Contile response cache control header is empty");
return 0;
}
const [, staleIfError] = cacheHeader.match(/stale-if-error=\s*([0-9]+)/i);
const [, maxAge] = cacheHeader.match(/max-age=\s*([0-9]+)/i);
const validFor =
Number.parseInt(staleIfError, 10) + Number.parseInt(maxAge, 10);
return isNaN(validFor) ? 0 : validFor;
}
/**
* Load Tiles from Contile Cache Prefs
*/
async _loadTilesFromCache() {
lazy.log.info("Contile client is trying to load tiles from local cache.");
const now = Math.round(Date.now() / 1000);
const lastFetch = Services.prefs.getIntPref(
CONTILE_CACHE_LAST_FETCH_PREF,
0
);
const validFor = Services.prefs.getIntPref(
CONTILE_CACHE_VALID_FOR_PREF,
CONTILE_CACHE_VALID_FOR_FALLBACK
);
this._topSitesFeed._telemetryUtility.setSponsoredTilesConfigured();
if (now <= lastFetch + validFor) {
try {
const cachedData = (await this.cache.get()) || {};
let cachedTiles = cachedData.contile;
this._topSitesFeed._telemetryUtility.setTiles(cachedTiles);
cachedTiles = this._filterBlockedSponsors(cachedTiles);
this._topSitesFeed._telemetryUtility.determineFilteredTilesAndSetToDismissed(
cachedTiles
);
this._sites = cachedTiles;
lazy.log.info("Local cache loaded.");
return true;
} catch (error) {
lazy.log.warn(`Failed to load tiles from local cache: ${error}.`);
return false;
}
}
return false;
}
/**
* Determine number of Tiles to get from Contile
*/
_getMaxNumFromContile() {
return (
lazy.NimbusFeatures.pocketNewtab.getVariable(
NIMBUS_VARIABLE_CONTILE_MAX_NUM_SPONSORED
) ?? CONTILE_MAX_NUM_SPONSORED
);
}
/**
* Normalize new Unified Ads API response into
* previous Contile ads response
*/
_normalizeTileData(data) {
const formattedTileData = [];
const responseTilesData = Object.values(data);
for (const tileData of responseTilesData) {
if (tileData?.length) {
// eslint-disable-next-line prefer-destructuring
const tile = tileData[0];
const formattedData = {
id: tile.block_key,
block_key: tile.block_key,
name: tile.name,
url: tile.url,
click_url: tile.callbacks.click,
image_url: tile.image_url,
impression_url: tile.callbacks.impression,
image_size: 200,
};
formattedTileData.push(formattedData);
}
}
return { tiles: formattedTileData };
}
// eslint-disable-next-line max-statements
async _fetchSites() {
if (
!lazy.NimbusFeatures.newtab.getVariable(
NIMBUS_VARIABLE_CONTILE_ENABLED
) ||
!this._topSitesFeed.store.getState().Prefs.values[SHOW_SPONSORED_PREF]
) {
if (this._sites.length) {
this._sites = [];
return true;
}
return false;
}
let response;
const state = this._topSitesFeed.store.getState();
const unifiedAdsTilesEnabled =
state.Prefs.values[PREF_UNIFIED_ADS_TILES_ENABLED];
const serviceName = unifiedAdsTilesEnabled ? "MARS" : "Contile";
try {
// Fetch tiles via MARS unified ads service
if (unifiedAdsTilesEnabled) {
const headers = new Headers();
headers.append("content-type", "application/json");
const endpointBaseUrl = state.Prefs.values[PREF_UNIFIED_ADS_ENDPOINT];
let blockedSponsors =
this._topSitesFeed.store.getState().Prefs.values[
PREF_UNIFIED_ADS_BLOCKED_LIST
];
// Overwrite URL to Unified Ads endpoint
const fetchUrl = `${endpointBaseUrl}v1/ads`;
const placementsArray = state.Prefs.values[
PREF_UNIFIED_ADS_PLACEMENTS
]?.split(`,`)
.map(s => s.trim())
.filter(item => item);
const countsArray = state.Prefs.values[PREF_UNIFIED_ADS_COUNTS]?.split(
`,`
)
.map(s => s.trim())
.filter(item => item)
.map(item => parseInt(item, 10));
response = await this._topSitesFeed.fetch(fetchUrl, {
method: "POST",
headers,
body: JSON.stringify({
context_id: lazy.contextId,
placements: placementsArray.map((placement, index) => ({
placement,
count: countsArray[index],
})),
blocks: blockedSponsors.split(","),
}),
});
} else {
// (Default) Fetch tiles via Contile service
const fetchUrl = Services.prefs.getStringPref(CONTILE_ENDPOINT_PREF);
let options = {
credentials: "omit",
};
response = await this._topSitesFeed.fetch(fetchUrl, options);
}
if (!response.ok) {
lazy.log.warn(
`${serviceName} endpoint returned unexpected status: ${response.status}`
);
if (response.status === 304 || response.status >= 500) {
return await this._loadTilesFromCache();
}
}
const lastFetch = Math.round(Date.now() / 1000);
Services.prefs.setIntPref(CONTILE_CACHE_LAST_FETCH_PREF, lastFetch);
this._topSitesFeed._telemetryUtility.setSponsoredTilesConfigured();
// Contile returns 204 indicating there is no content at the moment.
// If this happens, it will clear `this._sites` reset the cached tiles
// to an empty array.
if (response.status === 204) {
this._topSitesFeed._telemetryUtility.clearTilesForProvider(
SPONSORED_TILE_PARTNER_AMP
);
if (this._sites.length) {
this._sites = [];
await this.cache.set("contile", this._sites);
return true;
}
return false;
}
let body = await response.json();
if (unifiedAdsTilesEnabled) {
// Converts response into normalized tiles[] array
body = this._normalizeTileData(body);
}
if (body?.sov) {
this._sov = JSON.parse(atob(body.sov));
}
if (body?.tiles && Array.isArray(body.tiles)) {
const useAdditionalTiles = lazy.NimbusFeatures.newtab.getVariable(
NIMBUS_VARIABLE_ADDITIONAL_TILES
);
const maxNumFromContile = this._getMaxNumFromContile();
let { tiles } = body;
this._topSitesFeed._telemetryUtility.setTiles(tiles);
if (
useAdditionalTiles !== undefined &&
!useAdditionalTiles &&
tiles.length > maxNumFromContile
) {
tiles.length = maxNumFromContile;
this._topSitesFeed._telemetryUtility.determineFilteredTilesAndSetToOversold(
tiles
);
}
tiles = this._filterBlockedSponsors(tiles);
this._topSitesFeed._telemetryUtility.determineFilteredTilesAndSetToDismissed(
tiles
);
if (tiles.length > maxNumFromContile) {
lazy.log.info(`Remove unused links from ${serviceName}`);
tiles.length = maxNumFromContile;
this._topSitesFeed._telemetryUtility.determineFilteredTilesAndSetToOversold(
tiles
);
}
this._sites = tiles;
await this.cache.set("contile", this._sites);
if (!unifiedAdsTilesEnabled) {
Services.prefs.setIntPref(
CONTILE_CACHE_VALID_FOR_PREF,
this._extractCacheValidFor(
response.headers.get("cache-control") ||
response.headers.get("Cache-Control")
)
);
} else {
Services.prefs.setIntPref(
CONTILE_CACHE_VALID_FOR_PREF,
CONTILE_CACHE_VALID_FOR_FALLBACK
);
}
return true;
}
} catch (error) {
lazy.log.warn(
`Failed to fetch data from ${serviceName} server: ${error.message}`
);
return await this._loadTilesFromCache();
}
return false;
}
}
/**
* Creating a thin wrapper around PersistentCache.
* This makes it easier for us to write automated tests that simulate responses.
*/
ContileIntegration.prototype.PersistentCache = (...args) => {
return new lazy.PersistentCache(...args);
};
export class TopSitesFeed {
constructor() {
this._telemetryUtility = new TopSitesTelemetry();
this._contile = new ContileIntegration(this);
this._tippyTopProvider = new TippyTopProvider();
ChromeUtils.defineLazyGetter(
this,
"_currentSearchHostname",
getShortHostnameForCurrentSearch
);
this.dedupe = new Dedupe(this._dedupeKey);
this.frecentCache = new lazy.LinksCache(
lazy.NewTabUtils.activityStreamLinks,
"getTopSites",
CACHED_LINK_PROPS_TO_MIGRATE,
(oldOptions, newOptions) =>
// Refresh if no old options or requesting more items
!(oldOptions.numItems >= newOptions.numItems)
);
this.pinnedCache = new lazy.LinksCache(
lazy.NewTabUtils.pinnedLinks,
"links",
[...CACHED_LINK_PROPS_TO_MIGRATE, ...PINNED_FAVICON_PROPS_TO_MIGRATE]
);
lazy.PageThumbs.addExpirationFilter(this);
this._nimbusChangeListener = this._nimbusChangeListener.bind(this);
}
_nimbusChangeListener(event, reason) {
// The Nimbus API current doesn't specify the changed variable(s) in the
// listener callback, so we have to refresh unconditionally on every change
// of the `newtab` feature. It should be a manageable overhead given the
// current update cadence (6 hours) of Nimbus.
//
// Skip the experiment and rollout loading reasons since this feature has
// `isEarlyStartup` enabled, the feature variables are already available
// before the experiment or rollout loads.
if (
!["feature-experiment-loaded", "feature-rollout-loaded"].includes(reason)
) {
this._contile.refresh();
}
}
init() {
// If the feed was previously disabled PREFS_INITIAL_VALUES was never received
this._readDefaults({ isStartup: true });
this._storage = this.store.dbStorage.getDbTable("sectionPrefs");
this._contile.refresh();
Services.obs.addObserver(this, "browser-search-engine-modified");
Services.obs.addObserver(this, "browser-region-updated");
Services.prefs.addObserver(REMOTE_SETTING_DEFAULTS_PREF, this);
Services.prefs.addObserver(DEFAULT_SITES_OVERRIDE_PREF, this);
Services.prefs.addObserver(DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH, this);
lazy.NimbusFeatures.newtab.onUpdate(this._nimbusChangeListener);
}
uninit() {
lazy.PageThumbs.removeExpirationFilter(this);
Services.obs.removeObserver(this, "browser-search-engine-modified");
Services.obs.removeObserver(this, "browser-region-updated");
Services.prefs.removeObserver(REMOTE_SETTING_DEFAULTS_PREF, this);
Services.prefs.removeObserver(DEFAULT_SITES_OVERRIDE_PREF, this);
Services.prefs.removeObserver(DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH, this);
lazy.NimbusFeatures.newtab.offUpdate(this._nimbusChangeListener);
}
observe(subj, topic, data) {
switch (topic) {
case "browser-search-engine-modified":
// We should update the current top sites if the search engine has been changed since
// the search engine that gets filtered out of top sites has changed.
// We also need to drop search shortcuts when their engine gets removed / hidden.
if (
data === "engine-default" &&
this.store.getState().Prefs.values[FILTER_DEFAULT_SEARCH_PREF]
) {
delete this._currentSearchHostname;
this._currentSearchHostname = getShortHostnameForCurrentSearch();
}
this.refresh({ broadcast: true });
break;
case "browser-region-updated":
this._readDefaults();
break;
case "nsPref:changed":
if (
data === REMOTE_SETTING_DEFAULTS_PREF ||
data === DEFAULT_SITES_OVERRIDE_PREF ||
data.startsWith(DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH)
) {
this._readDefaults();
}
break;
}
}
_dedupeKey(site) {
return site && site.hostname;
}
/**
* _readDefaults - sets DEFAULT_TOP_SITES
*/
async _readDefaults({ isStartup = false } = {}) {
this._useRemoteSetting = false;
if (!Services.prefs.getBoolPref(REMOTE_SETTING_DEFAULTS_PREF)) {
this.refreshDefaults(
this.store.getState().Prefs.values[DEFAULT_SITES_PREF],
{ isStartup }
);
return;
}
// Try using default top sites from enterprise policies or tests. The pref
// is locked when set via enterprise policy. Tests have no default sites
// unless they set them via this pref.
if (
Services.prefs.prefIsLocked(DEFAULT_SITES_OVERRIDE_PREF) ||
Cu.isInAutomation
) {
let sites = Services.prefs.getStringPref(DEFAULT_SITES_OVERRIDE_PREF, "");
this.refreshDefaults(sites, { isStartup });
return;
}
// Clear out the array of any previous defaults.
DEFAULT_TOP_SITES.length = 0;
// Read defaults from contile.
const contileEnabled = lazy.NimbusFeatures.newtab.getVariable(
NIMBUS_VARIABLE_CONTILE_ENABLED
);
// Keep the number of positions in the array in sync with CONTILE_MAX_NUM_SPONSORED.
// sponsored_position is a 1-based index, and contilePositions is a 0-based index,
// so we need to add 1 to each of these.
// Also currently this does not work with SOV.
let contilePositions = lazy.NimbusFeatures.pocketNewtab
.getVariable(NIMBUS_VARIABLE_CONTILE_POSITIONS)
?.split(",")
.map(item => parseInt(item, 10) + 1)
.filter(item => !Number.isNaN(item));
if (!contilePositions || contilePositions.length === 0) {
contilePositions = [1, 2];
}
let hasContileTiles = false;
if (contileEnabled) {
let contilePositionIndex = 0;
// We need to loop through potential spocs and set their positions.
// If we run out of spocs or positions, we stop.
// First, we need to know which array is shortest. This is our exit condition.
const minLength = Math.min(
contilePositions.length,
this._contile.sites.length
);
// Loop until we run out of spocs or positions.
for (let i = 0; i < minLength; i++) {
let site = this._contile.sites[i];
let hostname = shortURL(site);
let link = {
isDefault: true,
url: site.url,
hostname,
sendAttributionRequest: false,
label: site.name,
show_sponsored_label: hostname !== "yandex",
sponsored_position: contilePositions[contilePositionIndex++],
sponsored_click_url: site.click_url,
sponsored_impression_url: site.impression_url,
sponsored_tile_id: site.id,
partner: SPONSORED_TILE_PARTNER_AMP,
block_key: site.id,
};
if (site.image_url && site.image_size >= MIN_FAVICON_SIZE) {
// Only use the image from Contile if it's hi-res, otherwise, fallback
// to the built-in favicons.
link.favicon = site.image_url;
link.faviconSize = site.image_size;
}
DEFAULT_TOP_SITES.push(link);
}
hasContileTiles = contilePositionIndex > 0;
//This is to catch where we receive 3 tiles but reduce to 2 early in the filtering, before blocked list applied.
this._telemetryUtility.determineFilteredTilesAndSetToOversold(
DEFAULT_TOP_SITES
);
}
// Read defaults from remote settings.
this._useRemoteSetting = true;
let remoteSettingData = await this._getRemoteConfig();
const sponsoredBlocklist = JSON.parse(
Services.prefs.getStringPref(TOP_SITES_BLOCKED_SPONSORS_PREF, "[]")
);
for (let siteData of remoteSettingData) {
let hostname = shortURL(siteData);
// Drop default sites when Contile already provided a sponsored one with
// the same host name.
if (
contileEnabled &&
DEFAULT_TOP_SITES.findIndex(site => site.hostname === hostname) > -1
) {
continue;
}
// Also drop those sponsored sites that were blocked by the user before
// with the same hostname.
if (
siteData.sponsored_position &&
sponsoredBlocklist.includes(hostname)
) {
continue;
}
let link = {
isDefault: true,
url: siteData.url,
hostname,
sendAttributionRequest: !!siteData.send_attribution_request,
};
if (siteData.url_urlbar_override) {
link.url_urlbar = siteData.url_urlbar_override;
}
if (siteData.title) {
link.label = siteData.title;
}
if (siteData.search_shortcut) {
link = await this.topSiteToSearchTopSite(link);
} else if (siteData.sponsored_position) {
if (contileEnabled && hasContileTiles) {
continue;
}
const {
sponsored_position,
sponsored_tile_id,
sponsored_impression_url,
sponsored_click_url,
block_key,
} = siteData;
link = {
sponsored_position,
sponsored_tile_id,
sponsored_impression_url,
sponsored_click_url,
block_key,
show_sponsored_label: link.hostname !== "yandex",
...link,
};
}
DEFAULT_TOP_SITES.push(link);
}
this.refresh({ broadcast: true, isStartup });
}
refreshDefaults(sites, { isStartup = false } = {}) {
// Clear out the array of any previous defaults
DEFAULT_TOP_SITES.length = 0;
// Add default sites if any based on the pref
if (sites) {
for (const url of sites.split(",")) {
const site = {
isDefault: true,
url,
};
site.hostname = shortURL(site);
DEFAULT_TOP_SITES.push(site);
}
}
this.refresh({ broadcast: true, isStartup });
}
async _getRemoteConfig(firstTime = true) {
if (!this._remoteConfig) {
this._remoteConfig = await lazy.RemoteSettings("top-sites");
this._remoteConfig.on("sync", () => {
this._readDefaults();
});
}
let result = [];
let failed = false;
try {
result = await this._remoteConfig.get();
} catch (ex) {
console.error(ex);
failed = true;
}
if (!result.length) {
console.error("Received empty top sites configuration!");
failed = true;
}
// If we failed, or the result is empty, try loading from the local dump.
if (firstTime && failed) {
await this._remoteConfig.db.clear();
// Now call this again.
return this._getRemoteConfig(false);
}
// Sort sites based on the "order" attribute.
result.sort((a, b) => a.order - b.order);
result = result.filter(topsite => {
// Filter by region.
if (topsite.exclude_regions?.includes(lazy.Region.home)) {
return false;
}
if (
topsite.include_regions?.length &&
!topsite.include_regions.includes(lazy.Region.home)
) {
return false;
}
// Filter by locale.
if (topsite.exclude_locales?.includes(Services.locale.appLocaleAsBCP47)) {
return false;
}
if (
topsite.include_locales?.length &&
!topsite.include_locales.includes(Services.locale.appLocaleAsBCP47)
) {
return false;
}
// Filter by experiment.
// Exclude this top site if any of the specified experiments are running.
if (
topsite.exclude_experiments?.some(experimentID =>
Services.prefs.getBoolPref(
DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH + experimentID,
false
)
)
) {
return false;
}
// Exclude this top site if none of the specified experiments are running.
if (
topsite.include_experiments?.length &&
topsite.include_experiments.every(
experimentID =>
!Services.prefs.getBoolPref(
DEFAULT_SITES_EXPERIMENTS_PREF_BRANCH + experimentID,
false
)
)
) {
return false;
}
return true;
});
return result;
}
filterForThumbnailExpiration(callback) {
const { rows } = this.store.getState().TopSites;
callback(
rows.reduce((acc, site) => {
acc.push(site.url);
if (site.customScreenshotURL) {
acc.push(site.customScreenshotURL);
}
return acc;
}, [])
);
}
/**
* shouldFilterSearchTile - is default filtering enabled and does a given hostname match the user's default search engine?
*
* @param {string} hostname a top site hostname, such as "amazon" or "foo"
* @returns {bool}
*/
shouldFilterSearchTile(hostname) {
if (
this.store.getState().Prefs.values[FILTER_DEFAULT_SEARCH_PREF] &&
(SEARCH_FILTERS.includes(hostname) ||
hostname === this._currentSearchHostname)
) {
return true;
}
return false;
}
/**
* _maybeInsertSearchShortcuts - if the search shortcuts experiment is running,
* insert search shortcuts if needed
* @param {Array} plainPinnedSites (from the pinnedSitesCache)
* @returns {Boolean} Did we insert any search shortcuts?
*/
async _maybeInsertSearchShortcuts(plainPinnedSites) {
// Only insert shortcuts if the experiment is running
if (this.store.getState().Prefs.values[SEARCH_SHORTCUTS_EXPERIMENT]) {
// We don't want to insert shortcuts we've previously inserted
const prevInsertedShortcuts = this.store
.getState()
.Prefs.values[SEARCH_SHORTCUTS_HAVE_PINNED_PREF].split(",")
.filter(s => s); // Filter out empty strings
const newInsertedShortcuts = [];
let shouldPin = this._useRemoteSetting
? DEFAULT_TOP_SITES.filter(s => s.searchTopSite).map(s => s.hostname)
: this.store
.getState()
.Prefs.values[SEARCH_SHORTCUTS_SEARCH_ENGINES_PREF].split(",");
shouldPin = shouldPin
.map(getSearchProvider)
.filter(s => s && s.shortURL !== this._currentSearchHostname);
// If we've previously inserted all search shortcuts return early
if (
shouldPin.every(shortcut =>
prevInsertedShortcuts.includes(shortcut.shortURL)
)
) {
return false;
}
const numberOfSlots =
this.store.getState().Prefs.values[ROWS_PREF] *
TOP_SITES_MAX_SITES_PER_ROW;
// The plainPinnedSites array is populated with pinned sites at their
// respective indices, and null everywhere else, but is not always the
// right length
const emptySlots = Math.max(numberOfSlots - plainPinnedSites.length, 0);
const pinnedSites = [...plainPinnedSites].concat(
Array(emptySlots).fill(null)
);
const tryToInsertSearchShortcut = async shortcut => {
const nextAvailable = pinnedSites.indexOf(null);
// Only add a search shortcut if the site isn't already pinned, we
// haven't previously inserted it, there's space to pin it, and the
// search engine is available in Firefox
if (
!pinnedSites.find(s => s && shortURL(s) === shortcut.shortURL) &&
!prevInsertedShortcuts.includes(shortcut.shortURL) &&
nextAvailable > -1 &&
(await checkHasSearchEngine(shortcut.keyword))
) {
const site = await this.topSiteToSearchTopSite({ url: shortcut.url });
this._pinSiteAt(site, nextAvailable);
pinnedSites[nextAvailable] = site;
newInsertedShortcuts.push(shortcut.shortURL);
}
};
for (let shortcut of shouldPin) {
await tryToInsertSearchShortcut(shortcut);
}
if (newInsertedShortcuts.length) {
this.store.dispatch(
ac.SetPref(
SEARCH_SHORTCUTS_HAVE_PINNED_PREF,
prevInsertedShortcuts.concat(newInsertedShortcuts).join(",")
)
);
return true;
}
}
return false;
}
/**
* This thin wrapper around global.fetch makes it easier for us to write
* automated tests that simulate responses from this fetch.
*/
fetch(...args) {
return fetch(...args);
}
/**
* Fetch topsites spocs from the DiscoveryStream feed.
*
* @returns {Array} An array of sponsored tile objects.
*/
fetchDiscoveryStreamSpocs() {
let sponsored = [];
const { DiscoveryStream } = this.store.getState();
if (DiscoveryStream) {
const discoveryStreamSpocs =
DiscoveryStream.spocs.data["sponsored-topsites"]?.items || [];
// Find the first component of a type and remove it from layout
const findSponsoredTopsitesPositions = name => {
for (const row of DiscoveryStream.layout) {
for (const component of row.components) {
if (component.placement?.name === name) {
return component.spocs.positions;
}
}
}
return null;
};
// Get positions from layout for now. This could be improved if we store position data in state.
const discoveryStreamSpocPositions =
findSponsoredTopsitesPositions("sponsored-topsites");
if (discoveryStreamSpocPositions?.length) {
function reformatImageURL(url, width, height) {
// Change the image URL to request a size tailored for the parent container width
// Also: force JPEG, quality 60, no upscaling, no EXIF data
// For now we wrap this in single quotes because this is being used in a url() css rule, and otherwise would cause a parsing error.
return `'https://img-getpocket.cdn.mozilla.net/${width}x${height}/filters:format(jpeg):quality(60):no_upscale():strip_exif()/${encodeURIComponent(
url
)}'`;
}
// We need to loop through potential spocs and set their positions.
// If we run out of spocs or positions, we stop.
// First, we need to know which array is shortest. This is our exit condition.
const minLength = Math.min(
discoveryStreamSpocPositions.length,
discoveryStreamSpocs.length
);
// Loop until we run out of spocs or positions.
for (let i = 0; i < minLength; i++) {
const positionIndex = discoveryStreamSpocPositions[i].index;
const spoc = discoveryStreamSpocs[i];
const link = {
favicon: reformatImageURL(spoc.raw_image_src, 96, 96),
faviconSize: 96,
type: "SPOC",
label: spoc.title || spoc.sponsor,
title: spoc.title || spoc.sponsor,
url: spoc.url,
flightId: spoc.flight_id,
id: spoc.id,
guid: spoc.id,
shim: spoc.shim,
// For now we are assuming position based on intended position.
// Actual position can shift based on other content.
// We send the intended position in the ping.
pos: positionIndex,
// Set this so that SPOC topsites won't be shown in the URL bar.
sponsored_position: positionIndex + 1,
// This is used for topsites deduping.
hostname: shortURL({ url: spoc.url }),
partner: SPONSORED_TILE_PARTNER_MOZ_SALES,
};
sponsored.push(link);
}
}
}
return sponsored;
}
// eslint-disable-next-line max-statements
async getLinksWithDefaults(isStartup = false) {
const prefValues = this.store.getState().Prefs.values;
const numItems = prefValues[ROWS_PREF] * TOP_SITES_MAX_SITES_PER_ROW;
const searchShortcutsExperiment = prefValues[SEARCH_SHORTCUTS_EXPERIMENT];
// We must wait for search services to initialize in order to access default
// search engine properties without triggering a synchronous initialization
try {
await Services.search.init();
} catch {
// We continue anyway because we want the user to see their sponsored,
// saved, or visited shortcut tiles even if search engines are not
// available.
}
// Get all frecent sites from history.
let frecent = [];
const cache = await this.frecentCache.request({
// We need to overquery due to the top 5 alexa search + default search possibly being removed
numItems: numItems + SEARCH_FILTERS.length + 1,
topsiteFrecency: FRECENCY_THRESHOLD,
});
for (let link of cache) {
const hostname = shortURL(link);
if (!this.shouldFilterSearchTile(hostname)) {
frecent.push({
...(searchShortcutsExperiment
? await this.topSiteToSearchTopSite(link)
: link),
hostname,
});
}
}
// Get defaults.
let contileSponsored = [];
let notBlockedDefaultSites = [];
for (let link of DEFAULT_TOP_SITES) {
// For sponsored Yandex links, default filtering is reversed: we only
// show them if Yandex is the default search engine.
if (link.sponsored_position && link.hostname === "yandex") {
if (link.hostname !== this._currentSearchHostname) {
continue;
}
} else if (this.shouldFilterSearchTile(link.hostname)) {
continue;
}
// Drop blocked default sites.
if (
lazy.NewTabUtils.blockedLinks.isBlocked({
url: link.url,
})
) {
continue;
}
// If we've previously blocked a search shortcut, remove the default top site
// that matches the hostname
const searchProvider = getSearchProvider(shortURL(link));
if (
searchProvider &&
lazy.NewTabUtils.blockedLinks.isBlocked({ url: searchProvider.url })
) {
continue;
}
if (link.sponsored_position) {
if (!prefValues[SHOW_SPONSORED_PREF]) {
continue;
}
contileSponsored[link.sponsored_position - 1] = link;
// Unpin search shortcut if present for the sponsored link to be shown
// instead.
this._unpinSearchShortcut(link.hostname);
} else {
notBlockedDefaultSites.push(
searchShortcutsExperiment
? await this.topSiteToSearchTopSite(link)
: link
);
}
}
this._telemetryUtility.determineFilteredTilesAndSetToDismissed(
contileSponsored
);
const discoverySponsored = this.fetchDiscoveryStreamSpocs();
this._telemetryUtility.setTiles(discoverySponsored);
const sponsored = this._mergeSponsoredLinks({
[SPONSORED_TILE_PARTNER_AMP]: contileSponsored,
[SPONSORED_TILE_PARTNER_MOZ_SALES]: discoverySponsored,
});
this._maybeCapSponsoredLinks(sponsored);
// This will set all extra tiles to oversold, including moz-sales.
this._telemetryUtility.determineFilteredTilesAndSetToOversold(sponsored);
// Get pinned links augmented with desired properties
let plainPinned = await this.pinnedCache.request();
// Insert search shortcuts if we need to.
// _maybeInsertSearchShortcuts returns true if any search shortcuts are
// inserted, meaning we need to expire and refresh the pinnedCache
if (await this._maybeInsertSearchShortcuts(plainPinned)) {
this.pinnedCache.expire();
plainPinned = await this.pinnedCache.request();
}
const pinned = await Promise.all(
plainPinned.map(async link => {
if (!link) {
return link;
}
// Drop pinned search shortcuts when their engine has been removed / hidden.
if (link.searchTopSite) {
const searchProvider = getSearchProvider(shortURL(link));
if (
!searchProvider ||
!(await checkHasSearchEngine(searchProvider.keyword))
) {
return null;
}
}
// Copy all properties from a frecent link and add more
const finder = other => other.url === link.url;
// Remove frecent link's screenshot if pinned link has a custom one
const frecentSite = frecent.find(finder);
if (frecentSite && link.customScreenshotURL) {
delete frecentSite.screenshot;
}
// If the link is a frecent site, do not copy over 'isDefault', else check
// if the site is a default site
const copy = Object.assign(
{},
frecentSite || { isDefault: !!notBlockedDefaultSites.find(finder) },
link,
{ hostname: shortURL(link) },
{ searchTopSite: !!link.searchTopSite }
);
// Add in favicons if we don't already have it
if (!copy.favicon) {
try {
lazy.NewTabUtils.activityStreamProvider._faviconBytesToDataURI(
await lazy.NewTabUtils.activityStreamProvider._addFavicons([copy])
);
for (const prop of PINNED_FAVICON_PROPS_TO_MIGRATE) {
copy.__sharedCache.updateLink(prop, copy[prop]);
}
} catch (e) {
// Some issue with favicon, so just continue without one
}
}
return copy;
})
);
// Remove any duplicates from frecent and default sites
const [, dedupedSponsored, dedupedFrecent, dedupedDefaults] =
this.dedupe.group(pinned, sponsored, frecent, notBlockedDefaultSites);
const dedupedUnpinned = [...dedupedFrecent, ...dedupedDefaults];
// Remove adult sites if we need to
const checkedAdult = lazy.FilterAdult.filter(dedupedUnpinned);
// Insert the original pinned sites into the deduped frecent and defaults.
let withPinned = insertPinned(checkedAdult, pinned);
// Insert sponsored sites at their desired position.
dedupedSponsored.forEach(link => {
if (!link) {
return;
}
let index = link.sponsored_position - 1;
if (index >= withPinned.length) {
withPinned[index] = link;
} else if (withPinned[index]?.sponsored_position) {
// We currently want DiscoveryStream spocs to replace existing spocs.
withPinned[index] = link;
} else {
withPinned.splice(index, 0, link);
}
});
// Remove excess items after we inserted sponsored ones.
withPinned = withPinned.slice(0, numItems);
// Now, get a tippy top icon, a rich icon, or screenshot for every item
for (const link of withPinned) {
if (link) {
// If there is a custom screenshot this is the only image we display
if (link.customScreenshotURL) {
this._fetchScreenshot(link, link.customScreenshotURL, isStartup);
} else if (link.searchTopSite && !link.isDefault) {
this._tippyTopProvider.processSite(link);
} else {
this._fetchIcon(link, isStartup);
}
// Remove internal properties that might be updated after dispatch
delete link.__sharedCache;
// Indicate that these links should get a frecency bonus when clicked
link.typedBonus = true;
}
}
this._linksWithDefaults = withPinned;
this._telemetryUtility.finalizeNewtabPingFields(dedupedSponsored);
return withPinned;
}
/**
* Cap sponsored links if they're more than the specified maximum.
*
* @param {Array} links An array of sponsored links. Capping will be performed in-place.
*/
_maybeCapSponsoredLinks(links) {
// Set maximum sponsored top sites
const maxSponsored =
lazy.NimbusFeatures.pocketNewtab.getVariable(
NIMBUS_VARIABLE_MAX_SPONSORED
) ?? MAX_NUM_SPONSORED;
if (links.length > maxSponsored) {
links.length = maxSponsored;
}
}
/**
* Merge sponsored links from all the partners using SOV if present.
* For each tile position, the user is assigned to one partner via stable sampling.
* If the chosen partner doesn't have a tile to serve, another tile from a different
* partner is used as the replacement.
*
* @param {Object} sponsoredLinks An object with sponsored links from all the partners.
* @returns {Array} An array of merged sponsored links.
*/
_mergeSponsoredLinks(sponsoredLinks) {
const { positions: allocatedPositions, ready: sovReady } =
this.store.getState().TopSites.sov || {};
if (
!this._contile.sov ||
!sovReady ||
!lazy.NimbusFeatures.pocketNewtab.getVariable(
NIMBUS_VARIABLE_CONTILE_SOV_ENABLED
)
) {
return Object.values(sponsoredLinks).flat();
}
// AMP links might have empty slots, remove them as SOV doesn't need those.
sponsoredLinks[SPONSORED_TILE_PARTNER_AMP] =
sponsoredLinks[SPONSORED_TILE_PARTNER_AMP].filter(Boolean);
let sponsored = [];
let chosenPartners = [];
for (const allocation of allocatedPositions) {
let link = null;
const { assignedPartner } = allocation;
if (assignedPartner) {
// Unknown partners are allowed so that new parters can be added to Shepherd
// sooner without waiting for client changes.
link = sponsoredLinks[assignedPartner]?.shift();
}
if (!link) {
// If the chosen partner doesn't have a tile for this postion, choose any
// one from another group. For simplicity, we do _not_ do resampling here
// against the remaining partners.
for (const partner of SPONSORED_TILE_PARTNERS) {
if (
partner === assignedPartner ||
sponsoredLinks[partner].length === 0
) {
continue;
}
link = sponsoredLinks[partner].shift();
break;
}
if (!link) {
// No more links to be added across all the partners, just return.
if (chosenPartners.length) {
Glean.newtab.sovAllocation.set(
chosenPartners.map(entry => JSON.stringify(entry))
);
}
return sponsored;
}
}
// Update the position fields. Note that postion is also 1-based in SOV.
link.sponsored_position = allocation.position;
if (link.pos !== undefined) {
// Pocket `pos` is 0-based.
link.pos = allocation.position - 1;
}
sponsored.push(link);
chosenPartners.push({
pos: allocation.position,
assigned: assignedPartner, // The assigned partner based on SOV
chosen: link.partner,
});
}
// Record chosen partners to glean
if (chosenPartners.length) {
Glean.newtab.sovAllocation.set(
chosenPartners.map(entry => JSON.stringify(entry))
);
}
// add the remaining contile sponsoredLinks when nimbus variable present
if (
lazy.NimbusFeatures.pocketNewtab.getVariable(
NIMBUS_VARIABLE_CONTILE_MAX_NUM_SPONSORED
)
) {
return sponsored.concat(sponsoredLinks[SPONSORED_TILE_PARTNER_AMP]);
}
return sponsored;
}
/**
* Refresh the top sites data for content.
* @param {bool} options.broadcast Should the update be broadcasted.
* @param {bool} options.isStartup Being called while TopSitesFeed is initting.
*/
async refresh(options = {}) {
if (!this._startedUp && !options.isStartup) {
// Initial refresh still pending.
return;
}
this._startedUp = true;
if (!this._tippyTopProvider.initialized) {
await this._tippyTopProvider.init();
}
const links = await this.getLinksWithDefaults({
isStartup: options.isStartup,
});
const newAction = { type: at.TOP_SITES_UPDATED, data: { links } };
let storedPrefs;
try {
storedPrefs = (await this._storage.get(SECTION_ID)) || {};
} catch (e) {
storedPrefs = {};
console.error("Problem getting stored prefs for TopSites");
}
newAction.data.pref = getDefaultOptions(storedPrefs);
if (options.isStartup) {
newAction.meta = {
isStartup: true,
};
}
if (options.broadcast) {
// Broadcast an update to all open content pages
this.store.dispatch(ac.BroadcastToContent(newAction));
} else {
// Don't broadcast only update the state and update the preloaded tab.
this.store.dispatch(ac.AlsoToPreloaded(newAction));
}
}
// Allocate ad positions to partners based on SOV via stable randomization.
async allocatePositions() {
// If the fetch to get sov fails for whatever reason, we can just return here.
// Code that uses this falls back to flattening allocations instead if this has failed.
if (!this._contile.sov) {
return;
}
// This sample input should ensure we return the same result for this allocation,
// even if called from other parts of the code.
const sampleInput = `${lazy.contextId}-${this._contile.sov.name}`;
const allocatedPositions = [];
for (const allocation of this._contile.sov.allocations) {
const allocatedPosition = {
position: allocation.position,
};
allocatedPositions.push(allocatedPosition);
const ratios = allocation.allocation.map(alloc => alloc.percentage);
if (ratios.length) {
const index = await lazy.Sampling.ratioSample(sampleInput, ratios);
allocatedPosition.assignedPartner =
allocation.allocation[index].partner;
}
}
this.store.dispatch(
ac.OnlyToMain({
type: at.SOV_UPDATED,
data: {
ready: !!allocatedPositions.length,
positions: allocatedPositions,
},
})
);
}
async updateCustomSearchShortcuts(isStartup = false) {
if (!this.store.getState().Prefs.values[SEARCH_SHORTCUTS_EXPERIMENT]) {
return;
}
if (!this._tippyTopProvider.initialized) {
await this._tippyTopProvider.init();
}
// Populate the state with available search shortcuts
let searchShortcuts = [];
for (const engine of await Services.search.getAppProvidedEngines()) {
const shortcut = CUSTOM_SEARCH_SHORTCUTS.find(s =>
engine.aliases.includes(s.keyword)
);
if (shortcut) {
let clone = { ...shortcut };
this._tippyTopProvider.processSite(clone);
searchShortcuts.push(clone);
}
}
this.store.dispatch(
ac.BroadcastToContent({
type: at.UPDATE_SEARCH_SHORTCUTS,
data: { searchShortcuts },
meta: {
isStartup,
},
})
);
}
async topSiteToSearchTopSite(site) {
const searchProvider = getSearchProvider(shortURL(site));
if (
!searchProvider ||
!(await checkHasSearchEngine(searchProvider.keyword))
) {
return site;
}
return {
...site,
searchTopSite: true,
label: searchProvider.keyword,
};
}
/**
* Get an image for the link preferring tippy top, rich favicon, screenshots.
*/
async _fetchIcon(link, isStartup = false) {
// Nothing to do if we already have a rich icon from the page
if (link.favicon && link.faviconSize >= MIN_FAVICON_SIZE) {
return;
}
// Nothing more to do if we can use a default tippy top icon
this._tippyTopProvider.processSite(link);
if (link.tippyTopIcon) {
return;
}
// Make a request for a better icon
this._requestRichIcon(link.url);
// Also request a screenshot if we don't have one yet
await this._fetchScreenshot(link, link.url, isStartup);
}
/**
* Fetch, cache and broadcast a screenshot for a specific topsite.
* @param link cached topsite object
* @param url where to fetch the image from
* @param isStartup Whether the screenshot is fetched while initting TopSitesFeed.
*/
async _fetchScreenshot(link, url, isStartup = false) {
// We shouldn't bother caching screenshots if they won't be shown.
if (
link.screenshot ||
!this.store.getState().Prefs.values[SHOWN_ON_NEWTAB_PREF]
) {
return;
}
await lazy.Screenshots.maybeCacheScreenshot(
link,
url,
"screenshot",
screenshot =>
this.store.dispatch(
ac.BroadcastToContent({
data: { screenshot, url: link.url },
type: at.SCREENSHOT_UPDATED,
meta: {
isStartup,
},
})
)
);
}
/**
* Dispatch screenshot preview to target or notify if request failed.
* @param customScreenshotURL {string} The URL used to capture the screenshot
* @param target {string} Id of content process where to dispatch the result
*/
async getScreenshotPreview(url, target) {
const preview = (await lazy.Screenshots.getScreenshotForURL(url)) || "";
this.store.dispatch(
ac.OnlyToOneContent(
{
data: { url, preview },
type: at.PREVIEW_RESPONSE,
},
target
)
);
}
_requestRichIcon(url) {
this.store.dispatch({
type: at.RICH_ICON_MISSING,
data: { url },
});
}
updateSectionPrefs(collapsed) {
this.store.dispatch(
ac.BroadcastToContent({
type: at.TOP_SITES_PREFS_UPDATED,
data: { pref: collapsed },
})
);
}
/**
* Inform others that top sites data has been updated due to pinned changes.
*/
_broadcastPinnedSitesUpdated() {
// Pinned data changed, so make sure we get latest
this.pinnedCache.expire();
// Refresh to update pinned sites with screenshots, trigger deduping, etc.
this.refresh({ broadcast: true });
}
/**
* Pin a site at a specific position saving only the desired keys.
* @param customScreenshotURL {string} User set URL of preview image for site
* @param label {string} User set string of custom site name
*/
async _pinSiteAt({ customScreenshotURL, label, url, searchTopSite }, index) {
const toPin = { url };
if (label) {
toPin.label = label;
}
if (customScreenshotURL) {
toPin.customScreenshotURL = customScreenshotURL;
}
if (searchTopSite) {
toPin.searchTopSite = searchTopSite;
}
lazy.NewTabUtils.pinnedLinks.pin(toPin, index);
await this._clearLinkCustomScreenshot({ customScreenshotURL, url });
}
async _clearLinkCustomScreenshot(site) {
// If screenshot url changed or was removed we need to update the cached link obj
if (site.customScreenshotURL !== undefined) {
const pinned = await this.pinnedCache.request();
const link = pinned.find(pin => pin && pin.url === site.url);
if (link && link.customScreenshotURL !== site.customScreenshotURL) {
link.__sharedCache.updateLink("screenshot", undefined);
}
}
}
/**
* Handle a pin action of a site to a position.
*/
async pin(action) {
let { site, index } = action.data;
index = this._adjustPinIndexForSponsoredLinks(site, index);
// If valid index provided, pin at that position
if (index >= 0) {
await this._pinSiteAt(site, index);
this._broadcastPinnedSitesUpdated();
} else {
// then we want to make sure to unblock that link if it has previously been
// blocked. We know if the site has been added because the index will be -1.
if (index === -1) {
lazy.NewTabUtils.blockedLinks.unblock({ url: site.url });
this.frecentCache.expire();
}
this.insert(action);
}
}
/**
* Handle an unpin action of a site.
*/
unpin(action) {
const { site } = action.data;
lazy.NewTabUtils.pinnedLinks.unpin(site);
this._broadcastPinnedSitesUpdated();
}
unpinAllSearchShortcuts() {
Services.prefs.clearUserPref(
`browser.newtabpage.activity-stream.${SEARCH_SHORTCUTS_HAVE_PINNED_PREF}`
);
for (let pinnedLink of lazy.NewTabUtils.pinnedLinks.links) {
if (pinnedLink && pinnedLink.searchTopSite) {
lazy.NewTabUtils.pinnedLinks.unpin(pinnedLink);
}
}
this.pinnedCache.expire();
}
_unpinSearchShortcut(vendor) {
for (let pinnedLink of lazy.NewTabUtils.pinnedLinks.links) {
if (
pinnedLink &&
pinnedLink.searchTopSite &&
shortURL(pinnedLink) === vendor
) {
lazy.NewTabUtils.pinnedLinks.unpin(pinnedLink);
this.pinnedCache.expire();
const prevInsertedShortcuts = this.store
.getState()
.Prefs.values[SEARCH_SHORTCUTS_HAVE_PINNED_PREF].split(",");
this.store.dispatch(
ac.SetPref(
SEARCH_SHORTCUTS_HAVE_PINNED_PREF,
prevInsertedShortcuts.filter(s => s !== vendor).join(",")
)
);
break;
}
}
}
/**
* Reduces the given pinning index by the number of preceding sponsored
* sites, to accomodate for sponsored sites pushing pinned ones to the side,
* effectively increasing their index again.
*/
_adjustPinIndexForSponsoredLinks(site, index) {
if (!this._linksWithDefaults) {
return index;
}
// Adjust insertion index for sponsored sites since their position is
// fixed.
let adjustedIndex = index;
for (let i = 0; i < index; i++) {
const link = this._linksWithDefaults[i];
if (
link &&
link.sponsored_position &&
this._linksWithDefaults[i]?.url !== site.url
) {
adjustedIndex--;
}
}
return adjustedIndex;
}
/**
* Insert a site to pin at a position shifting over any other pinned sites.
*/
_insertPin(site, originalIndex, draggedFromIndex) {
let index = this._adjustPinIndexForSponsoredLinks(site, originalIndex);
// Don't insert any pins past the end of the visible top sites. Otherwise,
// we can end up with a bunch of pinned sites that can never be unpinned again
// from the UI.
const topSitesCount =
this.store.getState().Prefs.values[ROWS_PREF] *
TOP_SITES_MAX_SITES_PER_ROW;
if (index >= topSitesCount) {
return;
}
let pinned = lazy.NewTabUtils.pinnedLinks.links;
if (!pinned[index]) {
this._pinSiteAt(site, index);
} else {
pinned[draggedFromIndex] = null;
// Find the hole to shift the pinned site(s) towards. We shift towards the
// hole left by the site being dragged.
let holeIndex = index;
const indexStep = index > draggedFromIndex ? -1 : 1;
while (pinned[holeIndex]) {
holeIndex += indexStep;
}
if (holeIndex >= topSitesCount || holeIndex < 0) {
// There are no holes, so we will effectively unpin the last slot and shifting
// towards it. This only happens when adding a new top site to an already
// fully pinned grid.
holeIndex = topSitesCount - 1;
}
// Shift towards the hole.
const shiftingStep = holeIndex > index ? -1 : 1;
while (holeIndex !== index) {
const nextIndex = holeIndex + shiftingStep;
this._pinSiteAt(pinned[nextIndex], holeIndex);
holeIndex = nextIndex;
}
this._pinSiteAt(site, index);
}
}
/**
* Handle an insert (drop/add) action of a site.
*/
async insert(action) {
let { index } = action.data;
// Treat invalid pin index values (e.g., -1, undefined) as insert in the first position
if (!(index > 0)) {
index = 0;
}
// Inserting a top site pins it in the specified slot, pushing over any link already
// pinned in the slot (unless it's the last slot, then it replaces).
this._insertPin(
action.data.site,
index,
action.data.draggedFromIndex !== undefined
? action.data.draggedFromIndex
: this.store.getState().Prefs.values[ROWS_PREF] *
TOP_SITES_MAX_SITES_PER_ROW
);
await this._clearLinkCustomScreenshot(action.data.site);
this._broadcastPinnedSitesUpdated();
}
updatePinnedSearchShortcuts({ addedShortcuts, deletedShortcuts }) {
// Unpin the deletedShortcuts.
deletedShortcuts.forEach(({ url }) => {
lazy.NewTabUtils.pinnedLinks.unpin({ url });
});
// Pin the addedShortcuts.
const numberOfSlots =
this.store.getState().Prefs.values[ROWS_PREF] *
TOP_SITES_MAX_SITES_PER_ROW;
addedShortcuts.forEach(shortcut => {
// Find first hole in pinnedLinks.
let index = lazy.NewTabUtils.pinnedLinks.links.findIndex(link => !link);
if (
index < 0 &&
lazy.NewTabUtils.pinnedLinks.links.length + 1 < numberOfSlots
) {
// pinnedLinks can have less slots than the total available.
index = lazy.NewTabUtils.pinnedLinks.links.length;
}
if (index >= 0) {
lazy.NewTabUtils.pinnedLinks.pin(shortcut, index);
} else {
// No slots available, we need to do an insert in first slot and push over other pinned links.
this._insertPin(shortcut, 0, numberOfSlots);
}
});
this._broadcastPinnedSitesUpdated();
}
onAction(action) {
switch (action.type) {
case at.INIT:
this.init();
this.updateCustomSearchShortcuts(true /* isStartup */);
break;
case at.DISCOVERY_STREAM_DEV_SYSTEM_TICK:
case at.SYSTEM_TICK:
this.refresh({ broadcast: false });
this._contile.periodicUpdate();
break;
// All these actions mean we need new top sites
case at.PLACES_HISTORY_CLEARED:
case at.PLACES_LINKS_DELETED:
this.frecentCache.expire();
this.refresh({ broadcast: true });
break;
case at.PLACES_LINKS_CHANGED:
this.frecentCache.expire();
this.refresh({ broadcast: false });
break;
case at.PLACES_LINK_BLOCKED:
this.frecentCache.expire();
this.pinnedCache.expire();
this.refresh({ broadcast: true });
break;
case at.PREF_CHANGED:
switch (action.data.name) {
case DEFAULT_SITES_PREF:
if (!this._useRemoteSetting) {
this.refreshDefaults(action.data.value);
}
break;
case ROWS_PREF:
case FILTER_DEFAULT_SEARCH_PREF:
case SEARCH_SHORTCUTS_SEARCH_ENGINES_PREF:
this.refresh({ broadcast: true });
break;
case SHOW_SPONSORED_PREF:
if (
lazy.NimbusFeatures.newtab.getVariable(
NIMBUS_VARIABLE_CONTILE_ENABLED
)
) {
this._contile.refresh();
} else {
this.refresh({ broadcast: true });
}
if (!action.data.value) {
this._contile._resetContileCache();
}
break;
case SEARCH_SHORTCUTS_EXPERIMENT:
if (action.data.value) {
this.updateCustomSearchShortcuts();
} else {
this.unpinAllSearchShortcuts();
}
this.refresh({ broadcast: true });
}
break;
case at.UPDATE_SECTION_PREFS:
if (action.data.id === SECTION_ID) {
this.updateSectionPrefs(action.data.value);
}
break;
case at.PREFS_INITIAL_VALUES:
if (!this._useRemoteSetting) {
this.refreshDefaults(action.data[DEFAULT_SITES_PREF]);
}
break;
case at.TOP_SITES_PIN:
this.pin(action);
break;
case at.TOP_SITES_UNPIN:
this.unpin(action);
break;
case at.TOP_SITES_INSERT:
this.insert(action);
break;
case at.PREVIEW_REQUEST:
this.getScreenshotPreview(action.data.url, action.meta.fromTarget);
break;
case at.UPDATE_PINNED_SEARCH_SHORTCUTS:
this.updatePinnedSearchShortcuts(action.data);
break;
case at.DISCOVERY_STREAM_SPOCS_UPDATE:
// Refresh to update sponsored topsites.
this.refresh({ broadcast: true, isStartup: action.meta.isStartup });
break;
case at.UNINIT:
this.uninit();
break;
}
}
}