Source code

Revision control

Copy as Markdown

Other Tools

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "CacheLog.h"
#include "CacheStorageService.h"
#include <iterator>
#include "CacheFileIOManager.h"
#include "CacheObserver.h"
#include "CacheIndex.h"
#include "CacheIndexIterator.h"
#include "CacheStorage.h"
#include "CacheEntry.h"
#include "CacheFileUtils.h"
#include "ErrorList.h"
#include "nsICacheStorageVisitor.h"
#include "nsIObserverService.h"
#include "nsIFile.h"
#include "nsIURI.h"
#include "nsINetworkPredictor.h"
#include "nsCOMPtr.h"
#include "nsContentUtils.h"
#include "nsNetCID.h"
#include "nsNetUtil.h"
#include "nsServiceManagerUtils.h"
#include "nsXULAppAPI.h"
#include "mozilla/AtomicBitfields.h"
#include "mozilla/TimeStamp.h"
#include "mozilla/DebugOnly.h"
#include "mozilla/glean/NetwerkMetrics.h"
#include "mozilla/Services.h"
#include "mozilla/StoragePrincipalHelper.h"
#include "mozilla/IntegerPrintfMacros.h"
#include "mozilla/glean/NetwerkCache2Metrics.h"
#include "mozilla/StaticPrefs_network.h"
namespace mozilla::net {
namespace {
void AppendMemoryStorageTag(nsAutoCString& key) {
// Using DEL as the very last ascii-7 character we can use in the list of
// attributes
key.Append('\x7f');
key.Append(',');
}
} // namespace
// Not defining as static or class member of CacheStorageService since
// it would otherwise need to include CacheEntry.h and that then would
// need to be exported to make nsNetModule.cpp compilable.
using GlobalEntryTables = nsClassHashtable<nsCStringHashKey, CacheEntryTable>;
/**
* Keeps tables of entries. There is one entries table for each distinct load
* context type. The distinction is based on following load context info
* states: <isPrivate|isAnon|inIsolatedMozBrowser> which builds a mapping
* key.
*
* Thread-safe to access, protected by the service mutex.
*/
static GlobalEntryTables* sGlobalEntryTables;
CacheMemoryConsumer::CacheMemoryConsumer(uint32_t aFlags) {
StoreFlags(aFlags);
}
void CacheMemoryConsumer::DoMemoryReport(uint32_t aCurrentSize) {
if (!(LoadFlags() & DONT_REPORT) && CacheStorageService::Self()) {
CacheStorageService::Self()->OnMemoryConsumptionChange(this, aCurrentSize);
}
}
CacheStorageService::MemoryPool::MemoryPool(EType aType) : mType(aType) {}
CacheStorageService::MemoryPool::~MemoryPool() {
if (mMemorySize != 0) {
NS_ERROR(
"Network cache reported memory consumption is not at 0, probably "
"leaking?");
}
}
uint32_t CacheStorageService::MemoryPool::Limit() const {
uint32_t limit = 0;
switch (mType) {
case DISK:
limit = CacheObserver::MetadataMemoryLimit();
break;
case MEMORY:
limit = CacheObserver::MemoryCacheCapacity();
break;
default:
MOZ_CRASH("Bad pool type");
}
static const uint32_t kMaxLimit = 0x3FFFFF;
if (limit > kMaxLimit) {
LOG((" a memory limit (%u) is unexpectedly high, clipping to %u", limit,
kMaxLimit));
limit = kMaxLimit;
}
return limit << 10;
}
NS_IMPL_ISUPPORTS(CacheStorageService, nsICacheStorageService,
nsIMemoryReporter, nsITimerCallback, nsICacheTesting,
nsINamed)
CacheStorageService* CacheStorageService::sSelf = nullptr;
CacheStorageService::CacheStorageService() {
CacheFileIOManager::Init();
MOZ_ASSERT(XRE_IsParentProcess());
MOZ_ASSERT(!sSelf);
sSelf = this;
sGlobalEntryTables = new GlobalEntryTables();
RegisterStrongMemoryReporter(this);
}
CacheStorageService::~CacheStorageService() {
LOG(("CacheStorageService::~CacheStorageService"));
sSelf = nullptr;
}
void CacheStorageService::Shutdown() {
mozilla::MutexAutoLock lock(mLock);
if (mShutdown) return;
LOG(("CacheStorageService::Shutdown - start"));
mShutdown = true;
nsCOMPtr<nsIRunnable> event =
NewRunnableMethod("net::CacheStorageService::ShutdownBackground", this,
&CacheStorageService::ShutdownBackground);
Dispatch(event);
#ifdef NS_FREE_PERMANENT_DATA
sGlobalEntryTables->Clear();
delete sGlobalEntryTables;
#endif
sGlobalEntryTables = nullptr;
LOG(("CacheStorageService::Shutdown - done"));
}
void CacheStorageService::ShutdownBackground() {
LOG(("CacheStorageService::ShutdownBackground - start"));
MOZ_ASSERT(IsOnManagementThread());
{
mozilla::MutexAutoLock lock(mLock);
// Cancel purge timer to avoid leaking.
if (mPurgeTimer) {
LOG((" freeing the timer"));
mPurgeTimer->Cancel();
}
}
#ifdef NS_FREE_PERMANENT_DATA
Pool(MemoryPool::EType::DISK).mManagedEntries.clear();
Pool(MemoryPool::EType::MEMORY).mManagedEntries.clear();
#endif
LOG(("CacheStorageService::ShutdownBackground - done"));
}
// Internal management methods
namespace {
// WalkCacheRunnable
// Base class for particular storage entries visiting
class WalkCacheRunnable : public Runnable,
public CacheStorageService::EntryInfoCallback {
protected:
WalkCacheRunnable(nsICacheStorageVisitor* aVisitor, bool aVisitEntries)
: Runnable("net::WalkCacheRunnable"),
mService(CacheStorageService::Self()),
mCallback(aVisitor) {
MOZ_ASSERT(NS_IsMainThread());
StoreNotifyStorage(true);
StoreVisitEntries(aVisitEntries);
}
virtual ~WalkCacheRunnable() {
if (mCallback) {
ProxyReleaseMainThread("WalkCacheRunnable::mCallback", mCallback);
}
}
RefPtr<CacheStorageService> mService;
nsCOMPtr<nsICacheStorageVisitor> mCallback;
uint64_t mSize{0};
// clang-format off
MOZ_ATOMIC_BITFIELDS(mAtomicBitfields, 8, (
(bool, NotifyStorage, 1),
(bool, VisitEntries, 1)
))
// clang-format on
Atomic<bool> mCancel{false};
};
// WalkMemoryCacheRunnable
// Responsible to visit memory storage and walk
// all entries on it asynchronously.
class WalkMemoryCacheRunnable : public WalkCacheRunnable {
public:
WalkMemoryCacheRunnable(nsILoadContextInfo* aLoadInfo, bool aVisitEntries,
nsICacheStorageVisitor* aVisitor)
: WalkCacheRunnable(aVisitor, aVisitEntries) {
CacheFileUtils::AppendKeyPrefix(aLoadInfo, mContextKey);
MOZ_ASSERT(NS_IsMainThread());
}
nsresult Walk() { return mService->Dispatch(this); }
private:
NS_IMETHOD Run() override {
if (CacheStorageService::IsOnManagementThread()) {
LOG(("WalkMemoryCacheRunnable::Run - collecting [this=%p]", this));
// First, walk, count and grab all entries from the storage
mozilla::MutexAutoLock lock(CacheStorageService::Self()->Lock());
if (!CacheStorageService::IsRunning()) return NS_ERROR_NOT_INITIALIZED;
// Count the entries to allocate the array memory all at once.
size_t numEntries = 0;
for (const auto& entries : sGlobalEntryTables->Values()) {
if (entries->Type() != CacheEntryTable::MEMORY_ONLY) {
continue;
}
numEntries += entries->Values().Count();
}
mEntryArray.SetCapacity(numEntries);
// Collect the entries.
for (const auto& entries : sGlobalEntryTables->Values()) {
if (entries->Type() != CacheEntryTable::MEMORY_ONLY) {
continue;
}
for (CacheEntry* entry : entries->Values()) {
MOZ_ASSERT(!entry->IsUsingDisk());
mSize += entry->GetMetadataMemoryConsumption();
int64_t size;
if (NS_SUCCEEDED(entry->GetDataSize(&size))) {
mSize += size;
}
mEntryArray.AppendElement(entry);
}
}
// Next, we dispatch to the main thread
} else if (NS_IsMainThread()) {
LOG(("WalkMemoryCacheRunnable::Run - notifying [this=%p]", this));
if (LoadNotifyStorage()) {
LOG((" storage"));
uint64_t capacity = CacheObserver::MemoryCacheCapacity();
capacity <<= 10; // kilobytes to bytes
// Second, notify overall storage info
mCallback->OnCacheStorageInfo(mEntryArray.Length(), mSize, capacity,
nullptr);
if (!LoadVisitEntries()) return NS_OK; // done
StoreNotifyStorage(false);
} else {
LOG((" entry [left=%zu, canceled=%d]", mEntryArray.Length(),
(bool)mCancel));
// Third, notify each entry until depleted or canceled.
if (mNextEntryIdx >= mEntryArray.Length() || mCancel) {
mCallback->OnCacheEntryVisitCompleted();
return NS_OK; // done
}
// Grab the next entry.
RefPtr<CacheEntry> entry = std::move(mEntryArray[mNextEntryIdx++]);
// Invokes this->OnEntryInfo, that calls the callback with all
// information of the entry.
CacheStorageService::GetCacheEntryInfo(entry, this);
}
} else {
MOZ_CRASH("Bad thread");
return NS_ERROR_FAILURE;
}
NS_DispatchToMainThread(this);
return NS_OK;
}
virtual ~WalkMemoryCacheRunnable() {
if (mCallback) {
ProxyReleaseMainThread("WalkMemoryCacheRunnable::mCallback", mCallback);
}
}
virtual void OnEntryInfo(const nsACString& aURISpec,
const nsACString& aIdEnhance, int64_t aDataSize,
int64_t aAltDataSize, uint32_t aFetchCount,
uint32_t aLastModifiedTime, uint32_t aExpirationTime,
bool aPinned, nsILoadContextInfo* aInfo) override {
nsresult rv;
nsCOMPtr<nsIURI> uri;
rv = NS_NewURI(getter_AddRefs(uri), aURISpec);
if (NS_FAILED(rv)) {
return;
}
rv = mCallback->OnCacheEntryInfo(uri, aIdEnhance, aDataSize, aAltDataSize,
aFetchCount, aLastModifiedTime,
aExpirationTime, aPinned, aInfo);
if (NS_FAILED(rv)) {
LOG((" callback failed, canceling the walk"));
mCancel = true;
}
}
private:
nsCString mContextKey;
nsTArray<RefPtr<CacheEntry>> mEntryArray;
size_t mNextEntryIdx{0};
};
// WalkDiskCacheRunnable
// Using the cache index information to get the list of files per context.
class WalkDiskCacheRunnable : public WalkCacheRunnable {
public:
WalkDiskCacheRunnable(nsILoadContextInfo* aLoadInfo, bool aVisitEntries,
nsICacheStorageVisitor* aVisitor)
: WalkCacheRunnable(aVisitor, aVisitEntries),
mLoadInfo(aLoadInfo),
mPass(COLLECT_STATS),
mCount(0) {}
nsresult Walk() {
// TODO, bug 998693
// Initial index build should be forced here so that about:cache soon
// after startup gives some meaningfull results.
// Dispatch to the INDEX level in hope that very recent cache entries
// information gets to the index list before we grab the index iterator
// for the first time. This tries to avoid miss of entries that has
// been created right before the visit is required.
RefPtr<CacheIOThread> thread = CacheFileIOManager::IOThread();
NS_ENSURE_TRUE(thread, NS_ERROR_NOT_INITIALIZED);
return thread->Dispatch(this, CacheIOThread::INDEX);
}
private:
// Invokes OnCacheEntryInfo callback for each single found entry.
// There is one instance of this class per one entry.
class OnCacheEntryInfoRunnable : public Runnable {
public:
explicit OnCacheEntryInfoRunnable(WalkDiskCacheRunnable* aWalker)
: Runnable("net::WalkDiskCacheRunnable::OnCacheEntryInfoRunnable"),
mWalker(aWalker) {}
NS_IMETHOD Run() override {
MOZ_ASSERT(NS_IsMainThread());
nsresult rv;
nsCOMPtr<nsIURI> uri;
rv = NS_NewURI(getter_AddRefs(uri), mURISpec);
if (NS_FAILED(rv)) {
return NS_OK;
}
rv = mWalker->mCallback->OnCacheEntryInfo(
uri, mIdEnhance, mDataSize, mAltDataSize, mFetchCount,
mLastModifiedTime, mExpirationTime, mPinned, mInfo);
if (NS_FAILED(rv)) {
mWalker->mCancel = true;
}
return NS_OK;
}
RefPtr<WalkDiskCacheRunnable> mWalker;
nsCString mURISpec;
nsCString mIdEnhance;
int64_t mDataSize{0};
int64_t mAltDataSize{0};
uint32_t mFetchCount{0};
uint32_t mLastModifiedTime{0};
uint32_t mExpirationTime{0};
bool mPinned{false};
nsCOMPtr<nsILoadContextInfo> mInfo;
};
NS_IMETHOD Run() override {
// The main loop
nsresult rv;
if (CacheStorageService::IsOnManagementThread()) {
switch (mPass) {
case COLLECT_STATS:
// Get quickly the cache stats.
uint32_t size;
rv = CacheIndex::GetCacheStats(mLoadInfo, &size, &mCount);
if (NS_FAILED(rv)) {
if (LoadVisitEntries()) {
// both onStorageInfo and onCompleted are expected
NS_DispatchToMainThread(this);
}
return NS_DispatchToMainThread(this);
}
mSize = static_cast<uint64_t>(size) << 10;
// Invoke onCacheStorageInfo with valid information.
NS_DispatchToMainThread(this);
if (!LoadVisitEntries()) {
return NS_OK; // done
}
mPass = ITERATE_METADATA;
[[fallthrough]];
case ITERATE_METADATA:
// Now grab the context iterator.
if (!mIter) {
rv =
CacheIndex::GetIterator(mLoadInfo, true, getter_AddRefs(mIter));
if (NS_FAILED(rv)) {
// Invoke onCacheEntryVisitCompleted now
return NS_DispatchToMainThread(this);
}
}
while (!mCancel && !CacheObserver::ShuttingDown()) {
if (CacheIOThread::YieldAndRerun()) return NS_OK;
SHA1Sum::Hash hash;
rv = mIter->GetNextHash(&hash);
if (NS_FAILED(rv)) break; // done (or error?)
// This synchronously invokes OnEntryInfo on this class where we
// redispatch to the main thread for the consumer callback.
CacheFileIOManager::GetEntryInfo(&hash, this);
}
// Invoke onCacheEntryVisitCompleted on the main thread
NS_DispatchToMainThread(this);
}
} else if (NS_IsMainThread()) {
if (LoadNotifyStorage()) {
nsCOMPtr<nsIFile> dir;
CacheFileIOManager::GetCacheDirectory(getter_AddRefs(dir));
uint64_t capacity = CacheObserver::DiskCacheCapacity();
capacity <<= 10; // kilobytes to bytes
mCallback->OnCacheStorageInfo(mCount, mSize, capacity, dir);
StoreNotifyStorage(false);
} else {
mCallback->OnCacheEntryVisitCompleted();
}
} else {
MOZ_CRASH("Bad thread");
return NS_ERROR_FAILURE;
}
return NS_OK;
}
virtual void OnEntryInfo(const nsACString& aURISpec,
const nsACString& aIdEnhance, int64_t aDataSize,
int64_t aAltDataSize, uint32_t aFetchCount,
uint32_t aLastModifiedTime, uint32_t aExpirationTime,
bool aPinned, nsILoadContextInfo* aInfo) override {
// Called directly from CacheFileIOManager::GetEntryInfo.
// Invoke onCacheEntryInfo on the main thread for this entry.
RefPtr<OnCacheEntryInfoRunnable> info = new OnCacheEntryInfoRunnable(this);
info->mURISpec = aURISpec;
info->mIdEnhance = aIdEnhance;
info->mDataSize = aDataSize;
info->mAltDataSize = aAltDataSize;
info->mFetchCount = aFetchCount;
info->mLastModifiedTime = aLastModifiedTime;
info->mExpirationTime = aExpirationTime;
info->mPinned = aPinned;
info->mInfo = aInfo;
NS_DispatchToMainThread(info);
}
RefPtr<nsILoadContextInfo> mLoadInfo;
enum {
// First, we collect stats for the load context.
COLLECT_STATS,
// Second, if demanded, we iterate over the entries gethered
// from the iterator and call CacheFileIOManager::GetEntryInfo
// for each found entry.
ITERATE_METADATA,
} mPass;
RefPtr<CacheIndexIterator> mIter;
uint32_t mCount;
};
} // namespace
void CacheStorageService::DropPrivateBrowsingEntries() {
mozilla::MutexAutoLock lock(mLock);
if (mShutdown) return;
nsTArray<nsCString> keys;
for (const nsACString& key : sGlobalEntryTables->Keys()) {
nsCOMPtr<nsILoadContextInfo> info = CacheFileUtils::ParseKey(key);
if (info && info->IsPrivate()) {
keys.AppendElement(key);
}
}
for (uint32_t i = 0; i < keys.Length(); ++i) {
DoomStorageEntries(keys[i], nullptr, true, false, nullptr);
}
}
// Helper methods
// static
bool CacheStorageService::IsOnManagementThread() {
RefPtr<CacheStorageService> service = Self();
if (!service) return false;
nsCOMPtr<nsIEventTarget> target = service->Thread();
if (!target) return false;
bool currentThread;
nsresult rv = target->IsOnCurrentThread(&currentThread);
return NS_SUCCEEDED(rv) && currentThread;
}
already_AddRefed<nsIEventTarget> CacheStorageService::Thread() const {
return CacheFileIOManager::IOTarget();
}
nsresult CacheStorageService::Dispatch(nsIRunnable* aEvent) {
RefPtr<CacheIOThread> cacheIOThread = CacheFileIOManager::IOThread();
if (!cacheIOThread) return NS_ERROR_NOT_AVAILABLE;
return cacheIOThread->Dispatch(aEvent, CacheIOThread::MANAGEMENT);
}
namespace CacheStorageEvictHelper {
nsresult ClearStorage(bool const aPrivate, bool const aAnonymous,
OriginAttributes& aOa) {
nsresult rv;
aOa.SyncAttributesWithPrivateBrowsing(aPrivate);
RefPtr<LoadContextInfo> info = GetLoadContextInfo(aAnonymous, aOa);
nsCOMPtr<nsICacheStorage> storage;
RefPtr<CacheStorageService> service = CacheStorageService::Self();
NS_ENSURE_TRUE(service, NS_ERROR_FAILURE);
// Clear disk storage
rv = service->DiskCacheStorage(info, getter_AddRefs(storage));
NS_ENSURE_SUCCESS(rv, rv);
rv = storage->AsyncEvictStorage(nullptr);
NS_ENSURE_SUCCESS(rv, rv);
// Clear memory storage
rv = service->MemoryCacheStorage(info, getter_AddRefs(storage));
NS_ENSURE_SUCCESS(rv, rv);
rv = storage->AsyncEvictStorage(nullptr);
NS_ENSURE_SUCCESS(rv, rv);
return NS_OK;
}
nsresult Run(OriginAttributes& aOa) {
nsresult rv;
// Clear all [private X anonymous] combinations
rv = ClearStorage(false, false, aOa);
NS_ENSURE_SUCCESS(rv, rv);
rv = ClearStorage(false, true, aOa);
NS_ENSURE_SUCCESS(rv, rv);
rv = ClearStorage(true, false, aOa);
NS_ENSURE_SUCCESS(rv, rv);
rv = ClearStorage(true, true, aOa);
NS_ENSURE_SUCCESS(rv, rv);
return NS_OK;
}
} // namespace CacheStorageEvictHelper
// nsICacheStorageService
NS_IMETHODIMP CacheStorageService::MemoryCacheStorage(
nsILoadContextInfo* aLoadContextInfo, nsICacheStorage** _retval) {
NS_ENSURE_ARG(_retval);
nsCOMPtr<nsICacheStorage> storage =
new CacheStorage(aLoadContextInfo, false, false, false);
storage.forget(_retval);
return NS_OK;
}
NS_IMETHODIMP CacheStorageService::DiskCacheStorage(
nsILoadContextInfo* aLoadContextInfo, nsICacheStorage** _retval) {
NS_ENSURE_ARG(_retval);
// TODO save some heap granularity - cache commonly used storages.
// When disk cache is disabled, still provide a storage, but just keep stuff
// in memory.
bool useDisk = CacheObserver::UseDiskCache();
nsCOMPtr<nsICacheStorage> storage = new CacheStorage(
aLoadContextInfo, useDisk, false /* size limit */, false /* don't pin */);
storage.forget(_retval);
return NS_OK;
}
NS_IMETHODIMP CacheStorageService::PinningCacheStorage(
nsILoadContextInfo* aLoadContextInfo, nsICacheStorage** _retval) {
NS_ENSURE_ARG(aLoadContextInfo);
NS_ENSURE_ARG(_retval);
// When disk cache is disabled don't pretend we cache.
if (!CacheObserver::UseDiskCache()) {
return NS_ERROR_NOT_AVAILABLE;
}
nsCOMPtr<nsICacheStorage> storage =
new CacheStorage(aLoadContextInfo, true /* use disk */,
true /* ignore size checks */, true /* pin */);
storage.forget(_retval);
return NS_OK;
}
NS_IMETHODIMP CacheStorageService::Clear() {
nsresult rv;
// Tell the index to block notification to AsyncGetDiskConsumption.
// Will be allowed again from CacheFileContextEvictor::EvictEntries()
// when all the context have been removed from disk.
CacheIndex::OnAsyncEviction(true);
mozilla::MutexAutoLock lock(mLock);
{
mozilla::MutexAutoLock forcedValidEntriesLock(mForcedValidEntriesLock);
mForcedValidEntries.Clear();
}
NS_ENSURE_TRUE(!mShutdown, NS_ERROR_NOT_INITIALIZED);
const auto keys = ToTArray<nsTArray<nsCString>>(sGlobalEntryTables->Keys());
for (const auto& key : keys) {
DoomStorageEntries(key, nullptr, true, false, nullptr);
}
// Passing null as a load info means to evict all contexts.
// EvictByContext() respects the entry pinning. EvictAll() does not.
rv = CacheFileIOManager::EvictByContext(nullptr, false, u""_ns);
NS_ENSURE_SUCCESS(rv, rv);
return NS_OK;
}
NS_IMETHODIMP CacheStorageService::ClearOrigin(nsIPrincipal* aPrincipal) {
nsresult rv;
if (NS_WARN_IF(!aPrincipal)) {
return NS_ERROR_FAILURE;
}
nsAutoString origin;
rv = nsContentUtils::GetWebExposedOriginSerialization(aPrincipal, origin);
NS_ENSURE_SUCCESS(rv, rv);
rv = ClearOriginInternal(origin, aPrincipal->OriginAttributesRef(), true);
NS_ENSURE_SUCCESS(rv, rv);
rv = ClearOriginInternal(origin, aPrincipal->OriginAttributesRef(), false);