Source code
Revision control
Copy as Markdown
Other Tools
/* vim: set ts=2 sts=2 et sw=2: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
#include <algorithm>
#include "Predictor.h"
#include "nsAppDirectoryServiceDefs.h"
#include "nsICacheStorage.h"
#include "nsICachingChannel.h"
#include "nsICancelable.h"
#include "nsIChannel.h"
#include "nsContentUtils.h"
#include "nsIDNSService.h"
#include "mozilla/dom/Document.h"
#include "nsIFile.h"
#include "nsIHttpChannel.h"
#include "nsIInputStream.h"
#include "nsILoadContext.h"
#include "nsILoadContextInfo.h"
#include "nsILoadGroup.h"
#include "nsINetworkPredictorVerifier.h"
#include "nsIObserverService.h"
#include "nsISpeculativeConnect.h"
#include "nsITimer.h"
#include "nsIURI.h"
#include "nsNetUtil.h"
#include "nsServiceManagerUtils.h"
#include "nsStreamUtils.h"
#include "nsString.h"
#include "nsThreadUtils.h"
#include "mozilla/Logging.h"
#include "mozilla/Components.h"
#include "mozilla/OriginAttributes.h"
#include "mozilla/Preferences.h"
#include "mozilla/SchedulerGroup.h"
#include "mozilla/StaticPrefs_network.h"
#include "mozilla/Telemetry.h"
#include "mozilla/net/NeckoCommon.h"
#include "mozilla/net/NeckoParent.h"
#include "LoadContextInfo.h"
#include "mozilla/ipc/URIUtils.h"
#include "SerializedLoadContext.h"
#include "mozilla/net/NeckoChild.h"
#include "mozilla/dom/ContentParent.h"
#include "mozilla/ClearOnShutdown.h"
#include "CacheControlParser.h"
#include "ReferrerInfo.h"
using namespace mozilla;
namespace mozilla {
namespace net {
Predictor* Predictor::sSelf = nullptr;
static LazyLogModule gPredictorLog("NetworkPredictor");
#define PREDICTOR_LOG(args) \
MOZ_LOG(gPredictorLog, mozilla::LogLevel::Debug, args)
#define NOW_IN_SECONDS() static_cast<uint32_t>(PR_Now() / PR_USEC_PER_SEC)
// All these time values are in sec
static const uint32_t ONE_DAY = 86400U;
static const uint32_t ONE_WEEK = 7U * ONE_DAY;
static const uint32_t ONE_MONTH = 30U * ONE_DAY;
static const uint32_t ONE_YEAR = 365U * ONE_DAY;
// Version of metadata entries we expect
static const uint32_t METADATA_VERSION = 1;
// Flags available in entries
// FLAG_PREFETCHABLE - we have determined that this item is eligible for
// prefetch
static const uint32_t FLAG_PREFETCHABLE = 1 << 0;
// We save 12 bits in the "flags" section of our metadata for actual flags, the
// rest are to keep track of a rolling count of which loads a resource has been
// used on to determine if we can prefetch that resource or not;
static const uint8_t kRollingLoadOffset = 12;
static const int32_t kMaxPrefetchRollingLoadCount = 20;
static const uint32_t kFlagsMask = ((1 << kRollingLoadOffset) - 1);
// ID Extensions for cache entries
#define PREDICTOR_ORIGIN_EXTENSION "predictor-origin"
// Get the full origin (scheme, host, port) out of a URI (maybe should be part
// of nsIURI instead?)
static nsresult ExtractOrigin(nsIURI* uri, nsIURI** originUri) {
nsAutoCString s;
nsresult rv = nsContentUtils::GetWebExposedOriginSerialization(uri, s);
NS_ENSURE_SUCCESS(rv, rv);
return NS_NewURI(originUri, s);
}
// All URIs we get passed *must* be http or https if they're not null. This
// helps ensure that.
static bool IsNullOrHttp(nsIURI* uri) {
if (!uri) {
return true;
}
return uri->SchemeIs("http") || uri->SchemeIs("https");
}
// Listener for the speculative DNS requests we'll fire off, which just ignores
// the result (since we're just trying to warm the cache). This also exists to
// reduce round-trips to the main thread, by being something threadsafe the
// Predictor can use.
NS_IMPL_ISUPPORTS(Predictor::DNSListener, nsIDNSListener);
NS_IMETHODIMP
Predictor::DNSListener::OnLookupComplete(nsICancelable* request,
nsIDNSRecord* rec, nsresult status) {
return NS_OK;
}
// Class to proxy important information from the initial predictor call through
// the cache API and back into the internals of the predictor. We can't use the
// predictor itself, as it may have multiple actions in-flight, and each action
// has different parameters.
NS_IMPL_ISUPPORTS(Predictor::Action, nsICacheEntryOpenCallback);
Predictor::Action::Action(bool fullUri, bool predict, Predictor::Reason reason,
nsIURI* targetURI, nsIURI* sourceURI,
nsINetworkPredictorVerifier* verifier,
Predictor* predictor)
: mFullUri(fullUri),
mPredict(predict),
mTargetURI(targetURI),
mSourceURI(sourceURI),
mVerifier(verifier),
mStackCount(0),
mPredictor(predictor) {
mStartTime = TimeStamp::Now();
if (mPredict) {
mPredictReason = reason.mPredict;
} else {
mLearnReason = reason.mLearn;
}
}
Predictor::Action::Action(bool fullUri, bool predict, Predictor::Reason reason,
nsIURI* targetURI, nsIURI* sourceURI,
nsINetworkPredictorVerifier* verifier,
Predictor* predictor, uint8_t stackCount)
: mFullUri(fullUri),
mPredict(predict),
mTargetURI(targetURI),
mSourceURI(sourceURI),
mVerifier(verifier),
mStackCount(stackCount),
mPredictor(predictor) {
mStartTime = TimeStamp::Now();
if (mPredict) {
mPredictReason = reason.mPredict;
} else {
mLearnReason = reason.mLearn;
}
}
NS_IMETHODIMP
Predictor::Action::OnCacheEntryCheck(nsICacheEntry* entry, uint32_t* result) {
*result = nsICacheEntryOpenCallback::ENTRY_WANTED;
return NS_OK;
}
NS_IMETHODIMP
Predictor::Action::OnCacheEntryAvailable(nsICacheEntry* entry, bool isNew,
nsresult result) {
MOZ_ASSERT(NS_IsMainThread(), "Got cache entry off main thread!");
nsAutoCString targetURI, sourceURI;
mTargetURI->GetAsciiSpec(targetURI);
if (mSourceURI) {
mSourceURI->GetAsciiSpec(sourceURI);
}
PREDICTOR_LOG(
("OnCacheEntryAvailable %p called. entry=%p mFullUri=%d mPredict=%d "
"mPredictReason=%d mLearnReason=%d mTargetURI=%s "
"mSourceURI=%s mStackCount=%d isNew=%d result=0x%08" PRIx32,
this, entry, mFullUri, mPredict, mPredictReason, mLearnReason,
targetURI.get(), sourceURI.get(), mStackCount, isNew,
static_cast<uint32_t>(result)));
if (NS_FAILED(result)) {
PREDICTOR_LOG(
("OnCacheEntryAvailable %p FAILED to get cache entry (0x%08" PRIX32
"). Aborting.",
this, static_cast<uint32_t>(result)));
return NS_OK;
}
Telemetry::AccumulateTimeDelta(Telemetry::PREDICTOR_WAIT_TIME, mStartTime);
if (mPredict) {
bool predicted =
mPredictor->PredictInternal(mPredictReason, entry, isNew, mFullUri,
mTargetURI, mVerifier, mStackCount);
Telemetry::AccumulateTimeDelta(Telemetry::PREDICTOR_PREDICT_WORK_TIME,
mStartTime);
if (predicted) {
Telemetry::AccumulateTimeDelta(
Telemetry::PREDICTOR_PREDICT_TIME_TO_ACTION, mStartTime);
} else {
Telemetry::AccumulateTimeDelta(
Telemetry::PREDICTOR_PREDICT_TIME_TO_INACTION, mStartTime);
}
} else {
mPredictor->LearnInternal(mLearnReason, entry, isNew, mFullUri, mTargetURI,
mSourceURI);
Telemetry::AccumulateTimeDelta(Telemetry::PREDICTOR_LEARN_WORK_TIME,
mStartTime);
}
return NS_OK;
}
NS_IMPL_ISUPPORTS(Predictor, nsINetworkPredictor, nsIObserver,
nsISpeculativeConnectionOverrider, nsIInterfaceRequestor,
nsICacheEntryMetaDataVisitor, nsINetworkPredictorVerifier)
Predictor::Predictor()
{
MOZ_ASSERT(!sSelf, "multiple Predictor instances!");
sSelf = this;
}
Predictor::~Predictor() {
if (mInitialized) Shutdown();
sSelf = nullptr;
}
// Predictor::nsIObserver
nsresult Predictor::InstallObserver() {
MOZ_ASSERT(NS_IsMainThread(), "Installing observer off main thread");
nsresult rv = NS_OK;
nsCOMPtr<nsIObserverService> obs = mozilla::services::GetObserverService();
if (!obs) {
return NS_ERROR_NOT_AVAILABLE;
}
rv = obs->AddObserver(this, NS_XPCOM_SHUTDOWN_OBSERVER_ID, false);
NS_ENSURE_SUCCESS(rv, rv);
return rv;
}
void Predictor::RemoveObserver() {
MOZ_ASSERT(NS_IsMainThread(), "Removing observer off main thread");
nsCOMPtr<nsIObserverService> obs = mozilla::services::GetObserverService();
if (obs) {
obs->RemoveObserver(this, NS_XPCOM_SHUTDOWN_OBSERVER_ID);
}
}
NS_IMETHODIMP
Predictor::Observe(nsISupports* subject, const char* topic,
const char16_t* data_unicode) {
nsresult rv = NS_OK;
MOZ_ASSERT(NS_IsMainThread(),
"Predictor observing something off main thread!");
if (!strcmp(NS_XPCOM_SHUTDOWN_OBSERVER_ID, topic)) {
Shutdown();
}
return rv;
}
// Predictor::nsISpeculativeConnectionOverrider
NS_IMETHODIMP
Predictor::GetIgnoreIdle(bool* ignoreIdle) {
*ignoreIdle = true;
return NS_OK;
}
NS_IMETHODIMP
Predictor::GetParallelSpeculativeConnectLimit(
uint32_t* parallelSpeculativeConnectLimit) {
*parallelSpeculativeConnectLimit = 6;
return NS_OK;
}
NS_IMETHODIMP
Predictor::GetIsFromPredictor(bool* isFromPredictor) {
*isFromPredictor = true;
return NS_OK;
}
NS_IMETHODIMP
Predictor::GetAllow1918(bool* allow1918) {
*allow1918 = false;
return NS_OK;
}
// Predictor::nsIInterfaceRequestor
NS_IMETHODIMP
Predictor::GetInterface(const nsIID& iid, void** result) {
return QueryInterface(iid, result);
}
// Predictor::nsICacheEntryMetaDataVisitor
#define SEEN_META_DATA "predictor::seen"
#define RESOURCE_META_DATA "predictor::resource-count"
#define META_DATA_PREFIX "predictor::"
static bool IsURIMetadataElement(const char* key) {
return StringBeginsWith(nsDependentCString(key),
nsLiteralCString(META_DATA_PREFIX)) &&
!nsLiteralCString(SEEN_META_DATA).Equals(key) &&
!nsLiteralCString(RESOURCE_META_DATA).Equals(key);
}
nsresult Predictor::OnMetaDataElement(const char* asciiKey,
const char* asciiValue) {
MOZ_ASSERT(NS_IsMainThread());
if (!IsURIMetadataElement(asciiKey)) {
// This isn't a bit of metadata we care about
return NS_OK;
}
nsCString key, value;
key.AssignASCII(asciiKey);
value.AssignASCII(asciiValue);
mKeysToOperateOn.AppendElement(key);
mValuesToOperateOn.AppendElement(value);
return NS_OK;
}
// Predictor::nsINetworkPredictor
nsresult Predictor::Init() {
MOZ_DIAGNOSTIC_ASSERT(!IsNeckoChild());
if (!NS_IsMainThread()) {
MOZ_ASSERT(false, "Predictor::Init called off the main thread!");
return NS_ERROR_UNEXPECTED;
}
nsresult rv = NS_OK;
rv = InstallObserver();
NS_ENSURE_SUCCESS(rv, rv);
mLastStartupTime = mStartupTime = NOW_IN_SECONDS();
if (!mDNSListener) {
mDNSListener = new DNSListener();
}
mCacheStorageService = mozilla::components::CacheStorage::Service(&rv);
NS_ENSURE_SUCCESS(rv, rv);
mSpeculativeService = mozilla::components::IO::Service(&rv);
NS_ENSURE_SUCCESS(rv, rv);
NS_ENSURE_SUCCESS(rv, rv);
mDnsService = mozilla::components::DNS::Service(&rv);
NS_ENSURE_SUCCESS(rv, rv);
mInitialized = true;
return rv;
}
namespace {
class PredictorLearnRunnable final : public Runnable {
public:
PredictorLearnRunnable(nsIURI* targetURI, nsIURI* sourceURI,
PredictorLearnReason reason,
const OriginAttributes& oa)
: Runnable("PredictorLearnRunnable"),
mTargetURI(targetURI),
mSourceURI(sourceURI),
mReason(reason),
mOA(oa) {
MOZ_DIAGNOSTIC_ASSERT(targetURI, "Must have a target URI");
}
~PredictorLearnRunnable() = default;
NS_IMETHOD Run() override {
if (!gNeckoChild) {
// This may have gone away between when this runnable was dispatched and
// when it actually runs, so let's be safe here, even though we asserted
// earlier.
PREDICTOR_LOG(("predictor::learn (async) gNeckoChild went away"));
return NS_OK;
}
PREDICTOR_LOG(("predictor::learn (async) forwarding to parent"));
gNeckoChild->SendPredLearn(mTargetURI, mSourceURI, mReason, mOA);
return NS_OK;
}
private:
nsCOMPtr<nsIURI> mTargetURI;
nsCOMPtr<nsIURI> mSourceURI;
PredictorLearnReason mReason;
const OriginAttributes mOA;
};
} // namespace
void Predictor::Shutdown() {
if (!NS_IsMainThread()) {
MOZ_ASSERT(false, "Predictor::Shutdown called off the main thread!");
return;
}
RemoveObserver();
mInitialized = false;
}
nsresult Predictor::Create(const nsIID& aIID, void** aResult) {
MOZ_ASSERT(NS_IsMainThread());
nsresult rv;
RefPtr<Predictor> svc = new Predictor();
if (IsNeckoChild()) {
NeckoChild::InitNeckoChild();
// Child threads only need to be call into the public interface methods
// so we don't bother with initialization
return svc->QueryInterface(aIID, aResult);
}
rv = svc->Init();
if (NS_FAILED(rv)) {
PREDICTOR_LOG(("Failed to initialize predictor, predictor will be a noop"));
}
// We treat init failure the same as the service being disabled, since this
// is all an optimization anyway. No need to freak people out. That's why we
// gladly continue on QI'ing here.
rv = svc->QueryInterface(aIID, aResult);
return rv;
}
NS_IMETHODIMP
Predictor::Predict(nsIURI* targetURI, nsIURI* sourceURI,
PredictorPredictReason reason,
JS::Handle<JS::Value> originAttributes,
nsINetworkPredictorVerifier* verifier, JSContext* aCx) {
OriginAttributes attrs;
if (!originAttributes.isObject() || !attrs.Init(aCx, originAttributes)) {
return NS_ERROR_INVALID_ARG;
}
return PredictNative(targetURI, sourceURI, reason, attrs, verifier);
}
// Called from the main thread to initiate predictive actions
NS_IMETHODIMP
Predictor::PredictNative(nsIURI* targetURI, nsIURI* sourceURI,
PredictorPredictReason reason,
const OriginAttributes& originAttributes,
nsINetworkPredictorVerifier* verifier) {
MOZ_ASSERT(NS_IsMainThread(),
"Predictor interface methods must be called on the main thread");
PREDICTOR_LOG(("Predictor::Predict"));
if (!StaticPrefs::network_predictor_enabled()) {
PREDICTOR_LOG((" not enabled"));
return NS_OK;
}
if (IsNeckoChild()) {
if (!gNeckoChild) {
return NS_ERROR_FAILURE;
}
PREDICTOR_LOG((" called on child process"));
// If two different threads are predicting concurently, this will be
// overwritten. Thankfully, we only use this in tests, which will
// overwrite mVerifier perhaps multiple times for each individual test;
// however, within each test, the multiple predict calls should have the
// same verifier.
if (verifier) {
PREDICTOR_LOG((" was given a verifier"));
mChildVerifier = verifier;
}
PREDICTOR_LOG((" forwarding to parent process"));
gNeckoChild->SendPredPredict(targetURI, sourceURI, reason, originAttributes,
verifier);
return NS_OK;
}
PREDICTOR_LOG((" called on parent process"));
if (!mInitialized) {
PREDICTOR_LOG((" not initialized"));
return NS_OK;
}
if (originAttributes.IsPrivateBrowsing()) {
// Don't want to do anything in PB mode
PREDICTOR_LOG((" in PB mode"));
return NS_OK;
}
if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) {
// Nothing we can do for non-HTTP[S] schemes
PREDICTOR_LOG((" got non-http[s] URI"));
return NS_OK;
}
// Ensure we've been given the appropriate arguments for the kind of
// prediction we're being asked to do
nsCOMPtr<nsIURI> uriKey = targetURI;
nsCOMPtr<nsIURI> originKey;
switch (reason) {
case nsINetworkPredictor::PREDICT_LINK:
if (!targetURI || !sourceURI) {
PREDICTOR_LOG((" link invalid URI state"));
return NS_ERROR_INVALID_ARG;
}
// Link hover is a special case where we can predict without hitting the
// db, so let's go ahead and fire off that prediction here.
PredictForLink(targetURI, sourceURI, originAttributes, verifier);
return NS_OK;
case nsINetworkPredictor::PREDICT_LOAD:
if (!targetURI || sourceURI) {
PREDICTOR_LOG((" load invalid URI state"));
return NS_ERROR_INVALID_ARG;
}
break;
case nsINetworkPredictor::PREDICT_STARTUP:
if (targetURI || sourceURI) {
PREDICTOR_LOG((" startup invalid URI state"));
return NS_ERROR_INVALID_ARG;
}
uriKey = mStartupURI;
originKey = mStartupURI;
break;
default:
PREDICTOR_LOG((" invalid reason"));
return NS_ERROR_INVALID_ARG;
}
Predictor::Reason argReason{};
argReason.mPredict = reason;
// First we open the regular cache entry, to ensure we don't gum up the works
// waiting on the less-important predictor-only cache entry
RefPtr<Predictor::Action> uriAction = new Predictor::Action(
Predictor::Action::IS_FULL_URI, Predictor::Action::DO_PREDICT, argReason,
targetURI, nullptr, verifier, this);
nsAutoCString uriKeyStr;
uriKey->GetAsciiSpec(uriKeyStr);
PREDICTOR_LOG((" Predict uri=%s reason=%d action=%p", uriKeyStr.get(),
reason, uriAction.get()));
nsCOMPtr<nsICacheStorage> cacheDiskStorage;
RefPtr<LoadContextInfo> lci = new LoadContextInfo(false, originAttributes);
nsresult rv = mCacheStorageService->DiskCacheStorage(
lci, getter_AddRefs(cacheDiskStorage));
NS_ENSURE_SUCCESS(rv, rv);
uint32_t openFlags =
nsICacheStorage::OPEN_READONLY | nsICacheStorage::OPEN_SECRETLY |
nsICacheStorage::OPEN_PRIORITY | nsICacheStorage::CHECK_MULTITHREADED;
cacheDiskStorage->AsyncOpenURI(uriKey, ""_ns, openFlags, uriAction);
// Now we do the origin-only (and therefore predictor-only) entry
nsCOMPtr<nsIURI> targetOrigin;
rv = ExtractOrigin(uriKey, getter_AddRefs(targetOrigin));
NS_ENSURE_SUCCESS(rv, rv);
if (!originKey) {
originKey = targetOrigin;
}
RefPtr<Predictor::Action> originAction = new Predictor::Action(
Predictor::Action::IS_ORIGIN, Predictor::Action::DO_PREDICT, argReason,
targetOrigin, nullptr, verifier, this);
nsAutoCString originKeyStr;
originKey->GetAsciiSpec(originKeyStr);
PREDICTOR_LOG((" Predict origin=%s reason=%d action=%p",
originKeyStr.get(), reason, originAction.get()));
openFlags = nsICacheStorage::OPEN_READONLY | nsICacheStorage::OPEN_SECRETLY |
nsICacheStorage::CHECK_MULTITHREADED;
cacheDiskStorage->AsyncOpenURI(originKey,
nsLiteralCString(PREDICTOR_ORIGIN_EXTENSION),
openFlags, originAction);
PREDICTOR_LOG((" predict returning"));
return NS_OK;
}
bool Predictor::PredictInternal(PredictorPredictReason reason,
nsICacheEntry* entry, bool isNew, bool fullUri,
nsIURI* targetURI,
nsINetworkPredictorVerifier* verifier,
uint8_t stackCount) {
MOZ_ASSERT(NS_IsMainThread());
PREDICTOR_LOG(("Predictor::PredictInternal"));
bool rv = false;
nsCOMPtr<nsILoadContextInfo> lci;
entry->GetLoadContextInfo(getter_AddRefs(lci));
if (!lci) {
return rv;
}
if (reason == nsINetworkPredictor::PREDICT_LOAD) {
MaybeLearnForStartup(targetURI, fullUri, *lci->OriginAttributesPtr());
}
if (isNew) {
// nothing else we can do here
PREDICTOR_LOG((" new entry"));
return rv;
}
switch (reason) {
case nsINetworkPredictor::PREDICT_LOAD:
rv = PredictForPageload(entry, targetURI, stackCount, fullUri, verifier);
break;
case nsINetworkPredictor::PREDICT_STARTUP:
rv = PredictForStartup(entry, fullUri, verifier);
break;
default:
PREDICTOR_LOG((" invalid reason"));
MOZ_ASSERT(false, "Got unexpected value for prediction reason");
}
return rv;
}
void Predictor::PredictForLink(nsIURI* targetURI, nsIURI* sourceURI,
const OriginAttributes& originAttributes,
nsINetworkPredictorVerifier* verifier) {
MOZ_ASSERT(NS_IsMainThread());
PREDICTOR_LOG(("Predictor::PredictForLink"));
if (!mSpeculativeService) {
PREDICTOR_LOG((" missing speculative service"));
return;
}
if (!StaticPrefs::network_predictor_enable_hover_on_ssl()) {
if (sourceURI->SchemeIs("https")) {
// We don't want to predict from an HTTPS page, to avoid info leakage
PREDICTOR_LOG((" Not predicting for link hover - on an SSL page"));
return;
}
}
nsCOMPtr<nsIPrincipal> principal =
BasePrincipal::CreateContentPrincipal(targetURI, originAttributes);
mSpeculativeService->SpeculativeConnect(targetURI, principal, nullptr, false);
if (verifier) {
PREDICTOR_LOG((" sending verification"));
verifier->OnPredictPreconnect(targetURI);
}
}
// This is the driver for prediction based on a new pageload.
static const uint8_t MAX_PAGELOAD_DEPTH = 10;
bool Predictor::PredictForPageload(nsICacheEntry* entry, nsIURI* targetURI,
uint8_t stackCount, bool fullUri,
nsINetworkPredictorVerifier* verifier) {
MOZ_ASSERT(NS_IsMainThread());
PREDICTOR_LOG(("Predictor::PredictForPageload"));
if (stackCount > MAX_PAGELOAD_DEPTH) {
PREDICTOR_LOG((" exceeded recursion depth!"));
return false;
}
uint32_t lastLoad;
nsresult rv = entry->GetLastFetched(&lastLoad);
NS_ENSURE_SUCCESS(rv, false);
int32_t globalDegradation = CalculateGlobalDegradation(lastLoad);
PREDICTOR_LOG((" globalDegradation = %d", globalDegradation));
uint32_t loadCount;
rv = entry->GetFetchCount(&loadCount);
NS_ENSURE_SUCCESS(rv, false);
nsCOMPtr<nsILoadContextInfo> lci;
rv = entry->GetLoadContextInfo(getter_AddRefs(lci));