Source code
Revision control
Copy as Markdown
Other Tools
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
import {
actionTypes as at,
actionCreators as ac,
const lazy = {};
ChromeUtils.defineESModuleGetters(lazy, {
});
export const STORIES_UPDATE_TIME = 30 * 60 * 1000; // 30 minutes
export const TOPICS_UPDATE_TIME = 3 * 60 * 60 * 1000; // 3 hours
const STORIES_NOW_THRESHOLD = 24 * 60 * 60 * 1000; // 24 hours
export const DEFAULT_RECS_EXPIRE_TIME = 60 * 60 * 1000; // 1 hour
export const SECTION_ID = "topstories";
const IMPRESSION_SOURCE = "TOP_STORIES";
export const SPOC_IMPRESSION_TRACKING_PREF =
"feeds.section.topstories.spoc.impressions";
const DISCOVERY_STREAM_PREF_ENABLED = "discoverystream.enabled";
const DISCOVERY_STREAM_PREF_ENABLED_PATH =
"browser.newtabpage.activity-stream.discoverystream.enabled";
export const REC_IMPRESSION_TRACKING_PREF =
"feeds.section.topstories.rec.impressions";
const PREF_USER_TOPSTORIES = "feeds.section.topstories";
const MAX_LIFETIME_CAP = 500; // Guard against misconfiguration on the server
const DISCOVERY_STREAM_PREF = "discoverystream.config";
export class TopStoriesFeed {
constructor(ds) {
// Use discoverystream config pref default values for fast path and
// if needed lazy load activity stream top stories feed based on
// actual user preference when INIT and PREF_CHANGED is invoked
this.discoveryStreamEnabled =
ds &&
ds.value &&
JSON.parse(ds.value).enabled &&
Services.prefs.getBoolPref(DISCOVERY_STREAM_PREF_ENABLED_PATH, false);
if (!this.discoveryStreamEnabled) {
this.initializeProperties();
}
}
initializeProperties() {
this.contentUpdateQueue = [];
this.spocCampaignMap = new Map();
this.cache = new PersistentCache(SECTION_ID, true);
this._prefs = new Prefs();
this.propertiesInitialized = true;
}
async onInit() {
SectionsManager.enableSection(SECTION_ID, true /* isStartup */);
if (this.discoveryStreamEnabled) {
return;
}
try {
const { options } = SectionsManager.sections.get(SECTION_ID);
const apiKey = this.getApiKeyFromPref(options.api_key_pref);
this.stories_endpoint = this.produceFinalEndpointUrl(
options.stories_endpoint,
apiKey
);
this.topics_endpoint = this.produceFinalEndpointUrl(
options.topics_endpoint,
apiKey
);
this.read_more_endpoint = options.read_more_endpoint;
this.stories_referrer = options.stories_referrer;
this.show_spocs = options.show_spocs;
this.storiesLastUpdated = 0;
this.topicsLastUpdated = 0;
this.storiesLoaded = false;
this.dispatchPocketCta(this._prefs.get("pocketCta"), false);
// Cache is used for new page loads, which shouldn't have changed data.
// If we have changed data, cache should be cleared,
// and last updated should be 0, and we can fetch.
let { stories, topics } = await this.loadCachedData();
if (this.storiesLastUpdated === 0) {
stories = await this.fetchStories();
}
if (this.topicsLastUpdated === 0) {
topics = await this.fetchTopics();
}
this.doContentUpdate({ stories, topics }, true);
this.storiesLoaded = true;
// This is filtered so an update function can return true to retry on the next run
this.contentUpdateQueue = this.contentUpdateQueue.filter(update =>
update()
);
} catch (e) {
console.error(`Problem initializing top stories feed: ${e.message}`);
}
}
init() {
SectionsManager.onceInitialized(this.onInit.bind(this));
}
async clearCache() {
await this.cache.set("stories", {});
await this.cache.set("topics", {});
await this.cache.set("spocs", {});
}
uninit() {
this.storiesLoaded = false;
SectionsManager.disableSection(SECTION_ID);
}
getPocketState(target) {
const action = {
type: at.POCKET_LOGGED_IN,
data: lazy.pktApi.isUserLoggedIn(),
};
this.store.dispatch(ac.OnlyToOneContent(action, target));
}
dispatchPocketCta(data, shouldBroadcast) {
const action = { type: at.POCKET_CTA, data: JSON.parse(data) };
this.store.dispatch(
shouldBroadcast
? ac.BroadcastToContent(action)
: ac.AlsoToPreloaded(action)
);
}
/**
* doContentUpdate - Updates topics and stories in the topstories section.
*
* Sections have one update action for the whole section.
* Redux creates a state race condition if you call the same action,
* twice, concurrently. Because of this, doContentUpdate is
* one place to update both topics and stories in a single action.
*
* Section updates used old topics if none are available,
* but clear stories if none are available. Because of this, if no
* stories are passed, we instead use the existing stories in state.
*
* @param {Object} This is an object with potential new stories or topics.
* @param {Boolean} shouldBroadcast If we should update existing tabs or not. For first page
* loads or pref changes, we want to update existing tabs,
* for system tick or other updates we do not.
*/
doContentUpdate({ stories, topics }, shouldBroadcast) {
let updateProps = {};
if (stories) {
updateProps.rows = stories;
} else {
const { Sections } = this.store.getState();
if (Sections && Sections.find) {
updateProps.rows = Sections.find(s => s.id === SECTION_ID).rows;
}
}
if (topics) {
Object.assign(updateProps, {
topics,
read_more_endpoint: this.read_more_endpoint,
});
}
// We should only be calling this once per init.
this.dispatchUpdateEvent(shouldBroadcast, updateProps);
}
async fetchStories() {
if (!this.stories_endpoint) {
return null;
}
try {
const response = await fetch(this.stories_endpoint, {
credentials: "omit",
});
if (!response.ok) {
throw new Error(
`Stories endpoint returned unexpected status: ${response.status}`
);
}
const body = await response.json();
this.updateSettings(body.settings);
this.stories = this.rotate(this.transform(body.recommendations));
this.cleanUpTopRecImpressionPref();
if (this.show_spocs && body.spocs) {
this.spocCampaignMap = new Map(
body.spocs.map(s => [s.id, `${s.campaign_id}`])
);
this.spocs = this.transform(body.spocs);
this.cleanUpCampaignImpressionPref();
}
this.storiesLastUpdated = Date.now();
body._timestamp = this.storiesLastUpdated;
this.cache.set("stories", body);
} catch (error) {
console.error(`Failed to fetch content: ${error.message}`);
}
return this.stories;
}
async loadCachedData() {
const data = await this.cache.get();
let stories = data.stories && data.stories.recommendations;
let topics = data.topics && data.topics.topics;
if (stories && !!stories.length && this.storiesLastUpdated === 0) {
this.updateSettings(data.stories.settings);
this.stories = this.rotate(this.transform(stories));
this.storiesLastUpdated = data.stories._timestamp;
if (data.stories.spocs && data.stories.spocs.length) {
this.spocCampaignMap = new Map(
data.stories.spocs.map(s => [s.id, `${s.campaign_id}`])
);
this.spocs = this.transform(data.stories.spocs);
this.cleanUpCampaignImpressionPref();
}
}
if (topics && !!topics.length && this.topicsLastUpdated === 0) {
this.topics = topics;
this.topicsLastUpdated = data.topics._timestamp;
}
return { topics: this.topics, stories: this.stories };
}
transform(items) {
if (!items) {
return [];
}
const calcResult = items
.filter(s => !lazy.NewTabUtils.blockedLinks.isBlocked({ url: s.url }))
.map(s => {
let mapped = {
guid: s.id,
hostname: s.domain || shortURL(Object.assign({}, s, { url: s.url })),
type:
Date.now() - s.published_timestamp * 1000 <= STORIES_NOW_THRESHOLD
? "now"
: "trending",
context: s.context,
icon: s.icon,
title: s.title,
description: s.excerpt,
image: this.normalizeUrl(s.image_src),
referrer: this.stories_referrer,
url: s.url,
score: s.item_score || 1,
spoc_meta: this.show_spocs
? { campaign_id: s.campaign_id, caps: s.caps }
: {},
};
// Very old cached spocs may not contain an `expiration_timestamp` property
if (s.expiration_timestamp) {
mapped.expiration_timestamp = s.expiration_timestamp;
}
return mapped;
})
.sort(this.compareScore);
return calcResult;
}
async fetchTopics() {
if (!this.topics_endpoint) {
return null;
}
try {
const response = await fetch(this.topics_endpoint, {
credentials: "omit",
});
if (!response.ok) {
throw new Error(
`Topics endpoint returned unexpected status: ${response.status}`
);
}
const body = await response.json();
const { topics } = body;
if (topics) {
this.topics = topics;
this.topicsLastUpdated = Date.now();
body._timestamp = this.topicsLastUpdated;
this.cache.set("topics", body);
}
} catch (error) {
console.error(`Failed to fetch topics: ${error.message}`);
}
return this.topics;
}
dispatchUpdateEvent(shouldBroadcast, data) {
SectionsManager.updateSection(SECTION_ID, data, shouldBroadcast);
}
compareScore(a, b) {
return b.score - a.score;
}
updateSettings(settings = {}) {
this.spocsPerNewTabs = settings.spocsPerNewTabs || 1; // Probability of a new tab getting a spoc [0,1]
this.recsExpireTime = settings.recsExpireTime;
}
// We rotate stories on the client so that
// active stories are at the front of the list, followed by stories that have expired
// impressions i.e. have been displayed for longer than recsExpireTime.
rotate(items) {
if (items.length <= 3) {
return items;
}
const maxImpressionAge = Math.max(
this.recsExpireTime * 1000 || DEFAULT_RECS_EXPIRE_TIME,
DEFAULT_RECS_EXPIRE_TIME
);
const impressions = this.readImpressionsPref(REC_IMPRESSION_TRACKING_PREF);
const expired = [];
const active = [];
for (const item of items) {
if (
impressions[item.guid] &&
Date.now() - impressions[item.guid] >= maxImpressionAge
) {
expired.push(item);
} else {
active.push(item);
}
}
return active.concat(expired);
}
getApiKeyFromPref(apiKeyPref) {
if (!apiKeyPref) {
return apiKeyPref;
}
return (
this._prefs.get(apiKeyPref) || Services.prefs.getCharPref(apiKeyPref)
);
}
produceFinalEndpointUrl(url, apiKey) {
if (!url) {
return url;
}
if (url.includes("$apiKey") && !apiKey) {
throw new Error(`An API key was specified but none configured: ${url}`);
}
return url.replace("$apiKey", apiKey);
}
// Need to remove parenthesis from image URLs as React will otherwise
// fail to render them properly as part of the card template.
normalizeUrl(url) {
if (url) {
return url.replace(/\(/g, "%28").replace(/\)/g, "%29");
}
return url;
}
shouldShowSpocs() {
return this.show_spocs && this.store.getState().Prefs.values.showSponsored;
}
dispatchSpocDone(target) {
const action = { type: at.POCKET_WAITING_FOR_SPOC, data: false };
this.store.dispatch(ac.OnlyToOneContent(action, target));
}
filterSpocs() {
if (!this.shouldShowSpocs()) {
return [];
}
if (Math.random() > this.spocsPerNewTabs) {
return [];
}
if (!this.spocs || !this.spocs.length) {
// We have stories but no spocs so there's nothing to do and this update can be
// removed from the queue.
return [];
}
// Filter spocs based on frequency caps
const impressions = this.readImpressionsPref(SPOC_IMPRESSION_TRACKING_PREF);
let spocs = this.spocs.filter(s =>
this.isBelowFrequencyCap(impressions, s)
);
// Filter out expired spocs based on `expiration_timestamp`
spocs = spocs.filter(spoc => {
// If cached data is so old it doesn't contain this property, assume the spoc is ok to show
if (!(`expiration_timestamp` in spoc)) {
return true;
}
// `expiration_timestamp` is the number of seconds elapsed since January 1, 1970 00:00:00 UTC
return spoc.expiration_timestamp * 1000 > Date.now();
});
return spocs;
}
maybeAddSpoc(target) {
const updateContent = () => {
let spocs = this.filterSpocs();
if (!spocs.length) {
this.dispatchSpocDone(target);
return false;
}
// Create a new array with a spoc inserted at index 2
const section = this.store
.getState()
.Sections.find(s => s.id === SECTION_ID);
let rows = section.rows.slice(0, this.stories.length);
rows.splice(2, 0, Object.assign(spocs[0], { pinned: true }));
// Send a content update to the target tab
const action = {
type: at.SECTION_UPDATE,
data: Object.assign({ rows }, { id: SECTION_ID }),
};
this.store.dispatch(ac.OnlyToOneContent(action, target));
this.dispatchSpocDone(target);
return false;
};
if (this.storiesLoaded) {
updateContent();
} else {
// Delay updating tab content until initial data has been fetched
this.contentUpdateQueue.push(updateContent);
}
}
// Frequency caps are based on campaigns, which may include multiple spocs.
// We currently support two types of frequency caps:
// - lifetime: Indicates how many times spocs from a campaign can be shown in total
// - period: Indicates how many times spocs from a campaign can be shown within a period
//
// So, for example, the feed configuration below defines that for campaign 1 no more
// than 5 spocs can be show in total, and no more than 2 per hour.
// "campaign_id": 1,
// "caps": {
// "lifetime": 5,
// "campaign": {
// "count": 2,
// "period": 3600
// }
// }
isBelowFrequencyCap(impressions, spoc) {
const campaignImpressions = impressions[spoc.spoc_meta.campaign_id];
if (!campaignImpressions) {
return true;
}
const lifeTimeCap = Math.min(
spoc.spoc_meta.caps && spoc.spoc_meta.caps.lifetime,
MAX_LIFETIME_CAP
);
const lifeTimeCapExceeded = campaignImpressions.length >= lifeTimeCap;
if (lifeTimeCapExceeded) {
return false;
}
const campaignCap =
(spoc.spoc_meta.caps && spoc.spoc_meta.caps.campaign) || {};
const campaignCapExceeded =
campaignImpressions.filter(
i => Date.now() - i < campaignCap.period * 1000
).length >= campaignCap.count;
return !campaignCapExceeded;
}
// Clean up campaign impression pref by removing all campaigns that are no
// longer part of the response, and are therefore considered inactive.
cleanUpCampaignImpressionPref() {
const campaignIds = new Set(this.spocCampaignMap.values());
this.cleanUpImpressionPref(
id => !campaignIds.has(id),
SPOC_IMPRESSION_TRACKING_PREF
);
}
// Clean up rec impression pref by removing all stories that are no
// longer part of the response.
cleanUpTopRecImpressionPref() {
const activeStories = new Set(this.stories.map(s => `${s.guid}`));
this.cleanUpImpressionPref(
id => !activeStories.has(id),
REC_IMPRESSION_TRACKING_PREF
);
}
/**
* Cleans up the provided impression pref (spocs or recs).
*
* @param isExpired predicate (boolean-valued function) that returns whether or not
* the impression for the given key is expired.
* @param pref the impression pref to clean up.
*/
cleanUpImpressionPref(isExpired, pref) {
const impressions = this.readImpressionsPref(pref);
let changed = false;
Object.keys(impressions).forEach(id => {
if (isExpired(id)) {
changed = true;
delete impressions[id];
}
});
if (changed) {
this.writeImpressionsPref(pref, impressions);
}
}
// Sets a pref mapping campaign IDs to timestamp arrays.
// The timestamps represent impressions which are used to calculate frequency caps.
recordCampaignImpression(campaignId) {
let impressions = this.readImpressionsPref(SPOC_IMPRESSION_TRACKING_PREF);
const timeStamps = impressions[campaignId] || [];
timeStamps.push(Date.now());
impressions = Object.assign(impressions, { [campaignId]: timeStamps });
this.writeImpressionsPref(SPOC_IMPRESSION_TRACKING_PREF, impressions);
}
// Sets a pref mapping story (rec) IDs to a single timestamp (time of first impression).
// We use these timestamps to guarantee a story doesn't stay on top for longer than
// configured in the feed settings (settings.recsExpireTime).
recordTopRecImpressions(topItems) {
let impressions = this.readImpressionsPref(REC_IMPRESSION_TRACKING_PREF);
let changed = false;
topItems.forEach(t => {
if (!impressions[t]) {
changed = true;
impressions = Object.assign(impressions, { [t]: Date.now() });
}
});
if (changed) {
this.writeImpressionsPref(REC_IMPRESSION_TRACKING_PREF, impressions);
}
}
readImpressionsPref(pref) {
const prefVal = this._prefs.get(pref);
return prefVal ? JSON.parse(prefVal) : {};
}
writeImpressionsPref(pref, impressions) {
this._prefs.set(pref, JSON.stringify(impressions));
}
async removeSpocs() {
// Quick hack so that SPOCS are removed from all open and preloaded tabs when
// they are disabled. The longer term fix should probably be to remove them
// in the Reducer.
await this.clearCache();
this.uninit();
this.init();
}
lazyLoadTopStories(options = {}) {
let { dsPref, userPref } = options;
if (!dsPref) {
dsPref = this.store.getState().Prefs.values[DISCOVERY_STREAM_PREF];
}
if (!userPref) {
userPref = this.store.getState().Prefs.values[PREF_USER_TOPSTORIES];
}
try {
this.discoveryStreamEnabled =
JSON.parse(dsPref).enabled &&
this.store.getState().Prefs.values[DISCOVERY_STREAM_PREF_ENABLED];
} catch (e) {
// Load activity stream top stories if fail to determine discovery stream state
this.discoveryStreamEnabled = false;
}
// Return without invoking initialization if top stories are loaded, or preffed off.
if (this.storiesLoaded || !userPref) {
return;
}
if (!this.discoveryStreamEnabled && !this.propertiesInitialized) {
this.initializeProperties();
}
this.init();
}
handleDisabled(action) {
switch (action.type) {
case at.INIT:
this.lazyLoadTopStories();
break;
case at.PREF_CHANGED:
if (action.data.name === DISCOVERY_STREAM_PREF) {
this.lazyLoadTopStories({ dsPref: action.data.value });
}
if (action.data.name === DISCOVERY_STREAM_PREF_ENABLED) {
this.lazyLoadTopStories();
}
if (action.data.name === PREF_USER_TOPSTORIES) {
if (action.data.value) {
// init topstories if value if true.
this.lazyLoadTopStories({ userPref: action.data.value });
} else {
this.uninit();
}
}
break;
case at.UNINIT:
this.uninit();
break;
}
}
async onAction(action) {
if (this.discoveryStreamEnabled) {
this.handleDisabled(action);
return;
}
switch (action.type) {
// Check discoverystream pref and load activity stream top stories only if needed
case at.INIT:
this.lazyLoadTopStories();
break;
case at.SYSTEM_TICK:
let stories;
let topics;
if (Date.now() - this.storiesLastUpdated >= STORIES_UPDATE_TIME) {
stories = await this.fetchStories();
}
if (Date.now() - this.topicsLastUpdated >= TOPICS_UPDATE_TIME) {
topics = await this.fetchTopics();
}
this.doContentUpdate({ stories, topics }, false);
break;
case at.UNINIT:
this.uninit();
break;
case at.NEW_TAB_REHYDRATED:
this.getPocketState(action.meta.fromTarget);
this.maybeAddSpoc(action.meta.fromTarget);
break;
case at.SECTION_OPTIONS_CHANGED:
if (action.data === SECTION_ID) {
await this.clearCache();
this.uninit();
this.init();
}
break;
case at.PLACES_LINK_BLOCKED:
if (this.spocs) {
this.spocs = this.spocs.filter(s => s.url !== action.data.url);
}
break;
case at.TELEMETRY_IMPRESSION_STATS: {
// We want to make sure we only track impressions from Top Stories,
// otherwise unexpected things that are not properly handled can happen.
// Example: Impressions from spocs on Discovery Stream can cause the
// Top Stories impressions pref to continuously grow, see bug #1523408
if (action.data.source === IMPRESSION_SOURCE) {
const payload = action.data;
const viewImpression = !(
"click" in payload ||
"block" in payload ||
"pocket" in payload
);
if (payload.tiles && viewImpression) {
if (this.shouldShowSpocs()) {
payload.tiles.forEach(t => {
if (this.spocCampaignMap.has(t.id)) {
this.recordCampaignImpression(this.spocCampaignMap.get(t.id));
}
});
}
const topRecs = payload.tiles
.filter(t => !this.spocCampaignMap.has(t.id))
.map(t => t.id);
this.recordTopRecImpressions(topRecs);
}
}
break;
}
case at.PREF_CHANGED:
if (action.data.name === DISCOVERY_STREAM_PREF) {
this.lazyLoadTopStories({ dsPref: action.data.value });
}
if (action.data.name === PREF_USER_TOPSTORIES) {
if (action.data.value) {
// init topstories if value if true.
this.lazyLoadTopStories({ userPref: action.data.value });
} else {
this.uninit();
}
}
// Check if spocs was disabled. Remove them if they were.
if (action.data.name === "showSponsored" && !action.data.value) {
await this.removeSpocs();
}
if (action.data.name === "pocketCta") {
this.dispatchPocketCta(action.data.value, true);
}
break;
}
}
}