firefox-main/dom/fs/test/xpcshell/test_orphanUsageCleanup.js (file symbol)

Source code

Revision control

Copy as Markdown

Other Tools

Test Info: Warnings

/**
* Any copyright is dedicated to the Public Domain.
*/
// Regression test for FileIds/Usages rows that nothing references anymore:
// rows whose handle was set to NULL because the on-disk blob could not be
// removed, and rows of temporary files whose writable stream never closed
// because of a crash. FileSystemDataManager removes every file id without a
// MainFiles row when it opens the database, and the Usages rows cascade.
//
// This test directly manipulates metadata.sqlite and therefore cannot use the
// shared module loader; it runs as a standalone privileged xpcshell test.
const { FileUtils } = ChromeUtils.importESModule(
);
const { PrincipalUtils } = ChromeUtils.importESModule(
);
const { QuotaUtils } = ChromeUtils.importESModule(
);
const { clearStoragesForOrigin, getCachedUsageForOrigin, resetStorage } =
ChromeUtils.importESModule(
);
const ORIGIN = "https://example.com";
const DB_PATH = "storage/default/https+++example.com/fs/metadata.sqlite";
// Usage recorded in the metadata.sqlite attached to bug 2034086 comment 0.
const PHANTOM_USAGE = 9_674_162_224;
async function evalInOPFSSandbox(principal, script) {
return new Promise(function (resolve, reject) {
const sandbox = new Cu.Sandbox(principal, {
wantGlobalProperties: ["storage"],
forceSecureContext: true,
});
sandbox.resolve = resolve;
sandbox.reject = reject;
Cu.evalInSandbox(script, sandbox);
});
}
function openOPFS(principal) {
return evalInOPFSSandbox(
principal,
`storage.getDirectory().then(resolve, reject);`
);
}
function createFile(principal, name) {
return evalInOPFSSandbox(
principal,
`storage
.getDirectory()
.then(root => root.getFileHandle(${JSON.stringify(name)}, { create: true }))
.then(resolve, reject);`
);
}
function queryInt(db, sql) {
const stmt = db.createStatement(sql);
try {
Assert.ok(stmt.executeStep(), `Query has a result: ${sql}`);
return stmt.getInt64(0);
} finally {
stmt.finalize();
}
}
add_task(async function test_orphan_usages_self_heal() {
const principal = PrincipalUtils.createPrincipal(ORIGIN);
info("Initializing OPFS origin and creating a file");
await createFile(principal, "kept.txt");
info("Closing the origin by resetting storage");
await resetStorage();
{
info("Injecting leftover FileIds/Usages rows");
const dbFile = FileUtils.getFile(DB_PATH);
Assert.ok(dbFile.exists(), "metadata.sqlite exists after OPFS use");
const db = Services.storage.openUnsharedDatabase(dbFile);
// A fake 32-byte fileId with no Files/Entries parent, the shape left
// behind when disk removal fails and handle is set to NULL.
const unlinkedId =
"X'DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF'";
db.executeSimpleSQL(
`INSERT INTO FileIds (fileId, handle) VALUES (${unlinkedId}, NULL)`
);
db.executeSimpleSQL(
`INSERT INTO Usages (handle, usage, tracked) VALUES (${unlinkedId}, ${PHANTOM_USAGE}, 0)`
);
// A temporary file of a writable stream whose close never happened: the
// fileId maps to an existing file but is not its main file.
const temporaryId =
"X'CAFEBABECAFEBABECAFEBABECAFEBABECAFEBABECAFEBABECAFEBABECAFEBABE'";
db.executeSimpleSQL(
`INSERT INTO FileIds (fileId, handle) ` +
`SELECT ${temporaryId}, handle FROM Files LIMIT 1`
);
db.executeSimpleSQL(
`INSERT INTO Usages (handle, usage, tracked) VALUES (${temporaryId}, 5000000, 1)`
);
const injected = queryInt(
db,
"SELECT count(*) FROM FileIds WHERE fileId NOT IN " +
"( SELECT fileId FROM MainFiles WHERE fileId IS NOT NULL )"
);
db.close();
Assert.equal(injected, 2, "Both leftover FileIds rows were injected");
}
await QuotaUtils.requestFinished(Services.qms.init());
await QuotaUtils.requestFinished(Services.qms.initTemporaryStorage());
Assert.ok(
FileUtils.getFile(DB_PATH).exists(),
"The origin survives temporary storage initialization"
);
info("Reopening OPFS origin, opening the database sweeps the leftovers");
await openOPFS(principal);
info("Closing origin again to read back DB cleanly");
await resetStorage();
{
info("Verifying leftover rows were removed");
const dbFile = FileUtils.getFile(DB_PATH);
const db = Services.storage.openUnsharedDatabase(dbFile);
const remaining = queryInt(
db,
"SELECT count(*) FROM FileIds WHERE fileId NOT IN " +
"( SELECT fileId FROM MainFiles WHERE fileId IS NOT NULL )"
);
const files = queryInt(db, "SELECT count(*) FROM Files");
const totalUsage = queryInt(
db,
"SELECT coalesce(sum(usage), 0) FROM Usages"
);
db.close();
Assert.equal(remaining, 0, "No FileIds rows without a main file remain");
Assert.equal(files, 1, "The real file is still there");
Assert.equal(totalUsage, 0, "Usages of the leftovers were cascade-deleted");
}
});
// Bug 2034086 comment 0 reported stale usage after an OPFS file disappeared.
// We first modeled it as a bare Usages row left behind when foreign key
// cascades did not run. The attached database actually retains its FileIds
// row with a NULL handle; the test above covers that state. This test keeps
// the bare-row case as a separate invalid state. On Android, the large stale
// usage can cause quota to evict the origin during temporary storage
// initialization, before OPFS opens and cleans up the row.
add_task(async function test_usages_without_fileids_self_heal() {
const principal = PrincipalUtils.createPrincipal(ORIGIN);
info("Clearing the origin so this task does not depend on the previous one");
await clearStoragesForOrigin(principal);
info("Initializing OPFS origin and creating a file");
await createFile(principal, "kept.txt");
info("Closing the origin by resetting storage");
await resetStorage();
{
info("Injecting a bare Usages row with no FileIds parent");
const dbFile = FileUtils.getFile(DB_PATH);
Assert.ok(dbFile.exists(), "metadata.sqlite exists after OPFS use");
const db = Services.storage.openUnsharedDatabase(dbFile);
// A 32-byte handle that exists only in Usages, not in FileIds.
const phantomHandle =
"X'8FA29A5FF357A0EE3EF1D9F481742462AED65C1F841E32E7ED9FB0A79E3E4881'";
db.executeSimpleSQL(
`INSERT INTO Usages (handle, usage, tracked) ` +
`VALUES (${phantomHandle}, ${PHANTOM_USAGE}, 0)`
);
const orphanedUsages = queryInt(
db,
"SELECT count(*) FROM Usages WHERE handle NOT IN " +
"( SELECT fileId FROM FileIds )"
);
db.close();
Assert.equal(orphanedUsages, 1, "Bare Usages row was injected");
}
await QuotaUtils.requestFinished(Services.qms.init());
await QuotaUtils.requestFinished(Services.qms.initTemporaryStorage());
Assert.ok(
FileUtils.getFile(DB_PATH).exists(),
"The origin survives temporary storage initialization"
);
{
const db = Services.storage.openUnsharedDatabase(
FileUtils.getFile(DB_PATH)
);
Assert.equal(
queryInt(db, "SELECT count(*) FROM Files"),
1,
"The real file survives temporary storage initialization"
);
db.close();
}
info("Reopening OPFS origin, the sweep removes the bare Usages row");
await openOPFS(principal);
const cachedUsage = await getCachedUsageForOrigin(principal);
Assert.less(
cachedUsage,
PHANTOM_USAGE,
"Cached quota usage no longer includes the phantom row"
);
info("Closing origin again to read back DB cleanly");
await resetStorage();
{
info("Verifying bare Usages row was removed");
const dbFile = FileUtils.getFile(DB_PATH);
const db = Services.storage.openUnsharedDatabase(dbFile);
const orphanedUsages = queryInt(
db,
"SELECT count(*) FROM Usages WHERE handle NOT IN " +
"( SELECT fileId FROM FileIds )"
);
const files = queryInt(db, "SELECT count(*) FROM Files");
const totalUsage = queryInt(
db,
"SELECT coalesce(sum(usage), 0) FROM Usages"
);
db.close();
Assert.equal(orphanedUsages, 0, "No Usages rows without a FileIds parent");
Assert.equal(files, 1, "The real file is still there");
Assert.equal(totalUsage, 0, "Phantom usage was removed");
}
});
async function testOrphanCleanup(cleanupOnOpen) {
const principal = PrincipalUtils.createPrincipal(ORIGIN);
await clearStoragesForOrigin(principal);
await createFile(principal, "kept.txt");
await evalInOPFSSandbox(
principal,
`(async () => {
const root = await storage.getDirectory();
const handle = await root.getFileHandle("kept.txt");
const writable = await handle.createWritable();
await writable.write("keep this data");
await writable.close();
})().then(resolve, reject);`
);
await resetStorage();
// These are the base32 filenames for the corresponding 32-byte file IDs.
const candidates = [
{ hex: "00".repeat(32), name: "A".repeat(52), hasFileId: true },
{
hex: "11".repeat(32),
name: "CEIRCEIRCEIRCEIRCEIRCEIRCEIRCEIRCEIRCEIRCEIRCEIRCEIQ",
hasFileId: true,
},
{ hex: "FF".repeat(32), name: "7".repeat(51) + "Q", hasFileId: false },
];
const dbFile = FileUtils.getFile(DB_PATH);
{
const db = Services.storage.openUnsharedDatabase(dbFile);
for (const [index, candidate] of candidates.entries()) {
const directory = dbFile.parent.clone();
directory.append(candidate.name.slice(0, 2));
await IOUtils.makeDirectory(directory.path);
const file = directory.clone();
file.append(candidate.name);
candidate.file = file;
if (index === 0 || cleanupOnOpen) {
// A directory at the blob path makes RemoveFile fail on every platform.
await IOUtils.makeDirectory(file.path);
} else {
await IOUtils.writeUTF8(file.path, "orphan");
}
if (candidate.hasFileId) {
db.executeSimpleSQL(
`INSERT INTO FileIds VALUES (X'${candidate.hex}', NULL)`
);
}
db.executeSimpleSQL(
`INSERT INTO Usages VALUES (X'${candidate.hex}', 6, 0)`
);
}
db.close();
}
await QuotaUtils.requestFinished(Services.qms.init());
await QuotaUtils.requestFinished(Services.qms.initTemporaryStorage());
await QuotaUtils.requestFinished(
Services.qms.initializeTemporaryOrigin("default", principal)
);
if (cleanupOnOpen) {
await resetStorage();
Services.prefs.setBoolPref("dom.quotaManager.loadQuotaFromCache", true);
Services.prefs.setBoolPref(
"dom.quotaManager.loadQuotaFromSecondaryCache",
true
);
await QuotaUtils.requestFinished(Services.qms.init());
await QuotaUtils.requestFinished(Services.qms.initTemporaryStorage());
await QuotaUtils.requestFinished(
Services.qms.initializeTemporaryOrigin("default", principal)
);
for (const candidate of candidates) {
Assert.ok(
candidate.file.exists(),
"Quota initialization retains blocked candidates"
);
}
for (const candidate of candidates.slice(1)) {
candidate.file.remove(false);
await IOUtils.writeUTF8(candidate.file.path, "orphan");
}
const usageBefore = await getCachedUsageForOrigin(principal);
await openOPFS(principal);
Assert.equal(
usageBefore - (await getCachedUsageForOrigin(principal)),
12,
"Database open releases only the successfully removed blobs' usage"
);
}
{
const db = Services.storage.openUnsharedDatabase(dbFile);
for (const [index, candidate] of candidates.entries()) {
const expected = index === 0 ? 1 : 0;
Assert.equal(candidate.file.exists(), Boolean(expected));
Assert.equal(
queryInt(
db,
`SELECT count(*) FROM FileIds WHERE fileId = X'${candidate.hex}'`
),
expected,
"Only the failed removal retains its FileIds row"
);
Assert.equal(
queryInt(
db,
`SELECT coalesce(sum(usage), 0) FROM Usages WHERE handle = X'${candidate.hex}'`
),
expected * 6,
"Only the failed removal retains its usage"
);
}
Assert.equal(queryInt(db, "SELECT count(*) FROM Files"), 1);
db.close();
}
await resetStorage();
const failed = candidates[0];
if (cleanupOnOpen) {
await QuotaUtils.requestFinished(Services.qms.init());
await QuotaUtils.requestFinished(Services.qms.initTemporaryStorage());
await QuotaUtils.requestFinished(
Services.qms.initializeTemporaryOrigin("default", principal)
);
}
failed.file.remove(false);
await IOUtils.writeUTF8(failed.file.path, "orphan");
await QuotaUtils.requestFinished(Services.qms.init());
await QuotaUtils.requestFinished(Services.qms.initTemporaryStorage());
await QuotaUtils.requestFinished(
Services.qms.initializeTemporaryOrigin("default", principal)
);
if (cleanupOnOpen) {
Assert.ok(failed.file.exists(), "The blob remains until database open");
const usageBefore = await getCachedUsageForOrigin(principal);
await openOPFS(principal);
Assert.equal(
usageBefore - (await getCachedUsageForOrigin(principal)),
6,
"Database open accounts for the successful retry"
);
}
Assert.ok(!failed.file.exists(), "The failed removal is retried");
{
const db = Services.storage.openUnsharedDatabase(dbFile);
Assert.equal(
queryInt(
db,
`SELECT count(*) FROM FileIds WHERE fileId = X'${failed.hex}'`
),
0
);
Assert.equal(
queryInt(
db,
`SELECT count(*) FROM Usages WHERE handle = X'${failed.hex}'`
),
0,
"The successful retry cascades to Usages"
);
db.close();
}
Assert.equal(
await evalInOPFSSandbox(
principal,
`(async () => {
const root = await storage.getDirectory();
const handle = await root.getFileHandle("kept.txt");
return (await handle.getFile()).text();
})().then(resolve, reject);`
),
"keep this data",
"Startup cleanup preserves the main file's contents"
);
}
add_task(async function test_startup_removes_blobs_and_retries_failures() {
await testOrphanCleanup(false);
});
add_task(
async function test_database_open_removes_blobs_and_retries_failures() {
try {
await testOrphanCleanup(true);
} finally {
Services.prefs.setBoolPref("dom.quotaManager.loadQuotaFromCache", false);
Services.prefs.setBoolPref(
"dom.quotaManager.loadQuotaFromSecondaryCache",
false
);
}
}
);