Source code
Revision control
Copy as Markdown
Other Tools
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
/**
* Handles serialization of the data and persistence into a file.
*
* This modules handles the raw data stored in JavaScript serializable objects,
* and contains no special validation or query logic, that is handled entirely
* by "storage.js" instead.
*
* The data can be manipulated only after it has been loaded from disk. The
* load process can happen asynchronously, through the "load" method, or
* synchronously, through "ensureDataReady". After any modification, the
* "saveSoon" method must be called to flush the data to disk asynchronously.
*
* The raw data should be manipulated synchronously, without waiting for the
* event loop or for promise resolution, so that the saved file is always
* consistent. This synchronous approach also simplifies the query and update
* logic. For example, it is possible to find an object and modify it
* immediately without caring whether other code modifies it in the meantime.
*
* An asynchronous shutdown observer makes sure that data is always saved before
* the browser is closed. The data cannot be modified during shutdown.
*
* The file is stored in JSON format, without indentation, using UTF-8 encoding.
*/
// Globals
const lazy = {};
ChromeUtils.defineESModuleGetters(lazy, {
});
ChromeUtils.defineLazyGetter(lazy, "gTextDecoder", function () {
return new TextDecoder();
});
const FileInputStream = Components.Constructor(
"@mozilla.org/network/file-input-stream;1",
"nsIFileInputStream",
"init"
);
/**
* Delay between a change to the data and the related save operation.
*/
const kSaveDelayMs = 1500;
/**
* Cleansed basenames of the filenames that telemetry can be recorded for.
* Keep synchronized with 'objects' from Events.yaml.
*/
const TELEMETRY_BASENAMES = new Set(["logins", "autofillprofiles"]);
// JSONFile
/**
* Handles serialization of the data and persistence into a file.
*
* @param config An object containing following members:
* - path: String containing the file path where data should be saved.
* - sanitizedBasename: Sanitized string identifier used for logging,
* shutdown debugging, and telemetry. Defaults to
* basename of given `path`, sanitized.
* - dataPostProcessor: Function triggered when data is just loaded. The
* data object will be passed as the first argument
* and should be returned no matter it's modified or
* not. Its failure leads to the failure of load()
* and ensureDataReady().
* - saveDelayMs: Number indicating the delay (in milliseconds) between a
* change to the data and the related save operation. The
* default value will be applied if omitted.
* - beforeSave: Promise-returning function triggered just before the
* data is written to disk. This can be used to create any
* intermediate directories before saving. The file will
* not be saved if the promise rejects or the function
* throws an exception.
* - finalizeAt: An `IOUtils` phase or barrier client that should
* automatically finalize the file when triggered. Defaults
* to `profileBeforeChange`; exposed as an option for
* testing.
* - compression: A compression algorithm to use when reading and
* writing the data.
* - backupTo: A string value indicating where writeAtomic should create
* a backup before writing to json files. Note that using this
* option currently ensures that we automatically restore backed
* up json files in load() and ensureDataReady() when original
* files are missing or corrupt.
*/
export function JSONFile(config) {
this.path = config.path;
this.sanitizedBasename =
config.sanitizedBasename ??
PathUtils.filename(this.path)
.replace(/\.json(.lz4)?$/, "")
.replaceAll(/[^a-zA-Z0-9_.]/g, "");
if (typeof config.dataPostProcessor === "function") {
this._dataPostProcessor = config.dataPostProcessor;
}
if (typeof config.beforeSave === "function") {
this._beforeSave = config.beforeSave;
}
if (config.saveDelayMs === undefined) {
config.saveDelayMs = kSaveDelayMs;
}
this._saver = new lazy.DeferredTask(() => this._save(), config.saveDelayMs);
this._options = {};
if (config.compression) {
this._options.decompress = this._options.compress = true;
}
if (config.backupTo) {
this._options.backupFile = this._options.backupTo = config.backupTo;
}
this._finalizeAt = config.finalizeAt || IOUtils.profileBeforeChange;
this._finalizeInternalBound = this._finalizeInternal.bind(this);
this._finalizeAt.addBlocker(
`JSON store: writing data for '${this.sanitizedBasename}'`,
this._finalizeInternalBound,
() => ({ sanitizedBasename: this.sanitizedBasename })
);
Services.telemetry.setEventRecordingEnabled("jsonfile", true);
}
JSONFile.prototype = {
/**
* String containing the file path where data should be saved.
*/
path: "",
/**
* Sanitized identifier used for logging, shutdown debugging, and telemetry.
*/
sanitizedBasename: "",
/**
* True when data has been loaded.
*/
dataReady: false,
/**
* DeferredTask that handles the save operation.
*/
_saver: null,
/**
* Internal data object.
*/
_data: null,
/**
* Internal fields used during finalization.
*/
_finalizeAt: null,
_finalizePromise: null,
_finalizeInternalBound: null,
/**
* Serializable object containing the data. This is populated directly with
* the data loaded from the file, and is saved without modifications.
*
* The raw data should be manipulated synchronously, without waiting for the
* event loop or for promise resolution, so that the saved file is always
* consistent.
*/
get data() {
if (!this.dataReady) {
throw new Error("Data is not ready.");
}
return this._data;
},
/**
* Sets the loaded data to a new object. This will overwrite any persisted
* data on the next save.
*/
set data(data) {
this._data = data;
this.dataReady = true;
},
/**
* Loads persistent data from the file to memory.
*
* @return {Promise}
* @resolves When the operation finished successfully.
* @rejects JavaScript exception when dataPostProcessor fails. It never fails
* if there is no dataPostProcessor.
*/
async load() {
if (this.dataReady) {
return;
}
let data = {};
try {
data = await IOUtils.readJSON(this.path, this._options);
// If synchronous loading happened in the meantime, exit now.
if (this.dataReady) {
return;
}
} catch (ex) {
// If an exception occurs because the file does not exist or it cannot be read,
// we do two things.
// 1. For consumers of JSONFile.sys.mjs that have configured a `backupTo` path option,
// we try to look for and use backed up json files first. If the backup
// is also not found or if the backup is unreadable, we then start with an empty file.
// 2. If a consumer does not configure a `backupTo` path option, we just start
// with an empty file.
// In the event that the file exists, but an exception is thrown because it cannot be read,
// we store it as a .corrupt file for debugging purposes.
let errorNo = ex.winLastError || ex.unixErrno;
this._recordTelemetry("load", errorNo ? errorNo.toString() : "");
if (!(DOMException.isInstance(ex) && ex.name == "NotFoundError")) {
console.error(ex);
// Move the original file to a backup location, ignoring errors.
try {
let uniquePath = await IOUtils.createUniqueFile(
PathUtils.parent(this.path),
PathUtils.filename(this.path) + ".corrupt",
0o600
);
await IOUtils.move(this.path, uniquePath);
this._recordTelemetry("load", "invalid_json");
} catch (e2) {
console.error(e2);
}
}
if (this._options.backupFile) {
// Restore the original file from the backup here so fresh writes to empty
// json files don't happen at any time in the future compromising the backup
// in the process.
try {
await IOUtils.copy(this._options.backupFile, this.path);
} catch (e) {
if (!(DOMException.isInstance(e) && e.name == "NotFoundError")) {
console.error(e);
}
}
try {
// We still read from the backup file here instead of the original file in case
// access to the original file is blocked, e.g. by anti-virus software on the
// user's computer.
data = await IOUtils.readJSON(
this._options.backupFile,
this._options
);
// If synchronous loading happened in the meantime, exit now.
if (this.dataReady) {
return;
}
this._recordTelemetry("load", "used_backup");
} catch (e3) {
if (!(DOMException.isInstance(e3) && e3.name == "NotFoundError")) {
console.error(e3);
}
}
}
// In some rare cases it's possible for data to have been added to
// our database between the call to IOUtils.read and when we've been
// notified that there was a problem with it. In that case, leave the
// synchronously-added data alone.
if (this.dataReady) {
return;
}
}
this._processLoadedData(data);
},
/**
* Loads persistent data from the file to memory, synchronously. An exception
* can be thrown only if dataPostProcessor exists and fails.
*/
ensureDataReady() {
if (this.dataReady) {
return;
}
let data = {};
try {
// This reads the file and automatically detects the UTF-8 encoding.
let inputStream = new FileInputStream(
new lazy.FileUtils.File(this.path),
lazy.FileUtils.MODE_RDONLY,
lazy.FileUtils.PERMS_FILE,
0
);
try {
let bytes = lazy.NetUtil.readInputStream(
inputStream,
inputStream.available()
);
data = JSON.parse(lazy.gTextDecoder.decode(bytes));
} finally {
inputStream.close();
}
} catch (ex) {
// If an exception occurs because the file does not exist or it cannot be read,
// we do two things.
// 1. For consumers of JSONFile.sys.mjs that have configured a `backupTo` path option,
// we try to look for and use backed up json files first. If the backup
// is also not found or if the backup is unreadable, we then start with an empty file.
// 2. If a consumer does not configure a `backupTo` path option, we just start
// with an empty file.
// In the event that the file exists, but an exception is thrown because it cannot be read,
// we store it as a .corrupt file for debugging purposes.
if (
!(
ex instanceof Components.Exception &&
ex.result == Cr.NS_ERROR_FILE_NOT_FOUND
)
) {
console.error(ex);
// Move the original file to a backup location, ignoring errors.
try {
let originalFile = new lazy.FileUtils.File(this.path);
let backupFile = originalFile.clone();
backupFile.leafName += ".corrupt";
backupFile.createUnique(
Ci.nsIFile.NORMAL_FILE_TYPE,
lazy.FileUtils.PERMS_FILE
);
backupFile.remove(false);
originalFile.moveTo(backupFile.parent, backupFile.leafName);
} catch (e2) {
console.error(e2);
}
}
if (this._options.backupFile) {
// Restore the original file from the backup here so fresh writes to empty
// json files don't happen at any time in the future compromising the backup
// in the process.
try {
let basename = PathUtils.filename(this.path);
let backupFile = new lazy.FileUtils.File(this._options.backupFile);
backupFile.copyTo(null, basename);
} catch (e) {
if (e.result != Cr.NS_ERROR_FILE_NOT_FOUND) {
console.error(e);
}
}
try {
// We still read from the backup file here instead of the original file in case
// access to the original file is blocked, e.g. by anti-virus software on the
// user's computer.
// This reads the file and automatically detects the UTF-8 encoding.
let inputStream = new FileInputStream(
new lazy.FileUtils.File(this._options.backupFile),
lazy.FileUtils.MODE_RDONLY,
lazy.FileUtils.PERMS_FILE,
0
);
try {
let bytes = lazy.NetUtil.readInputStream(
inputStream,
inputStream.available()
);
data = JSON.parse(lazy.gTextDecoder.decode(bytes));
} finally {
inputStream.close();
}
} catch (e3) {
if (e3.result != Cr.NS_ERROR_FILE_NOT_FOUND) {
console.error(e3);
}
}
}
}
this._processLoadedData(data);
},
/**
* Called when the data changed, this triggers asynchronous serialization.
*/
saveSoon() {
return this._saver.arm();
},
/**
* Saves persistent data from memory to the file.
*
* If an error occurs, the previous file is not deleted.
*
* @return {Promise}
* @resolves When the operation finished successfully.
* @rejects JavaScript exception.
*/
async _save() {
// Create or overwrite the file.
if (this._beforeSave) {
await Promise.resolve(this._beforeSave());
}
try {
await IOUtils.writeJSON(
this.path,
this._data,
Object.assign({ tmpPath: this.path + ".tmp" }, this._options)
);
} catch (ex) {
if (typeof this._data.toJSONSafe == "function") {
// If serialization fails, try fallback safe JSON converter.
await IOUtils.writeUTF8(
this.path,
this._data.toJSONSafe(),
Object.assign({ tmpPath: this.path + ".tmp" }, this._options)
);
}
}
},
/**
* Synchronously work on the data just loaded into memory.
*/
_processLoadedData(data) {
if (this._finalizePromise) {
// It's possible for `load` to race with `finalize`. In that case, don't
// process or set the loaded data.
return;
}
this.data = this._dataPostProcessor ? this._dataPostProcessor(data) : data;
},
_recordTelemetry(method, value) {
if (!TELEMETRY_BASENAMES.has(this.sanitizedBasename)) {
// Avoid recording so we don't log an error in the console.
return;
}
Services.telemetry.recordEvent(
"jsonfile",
method,
this.sanitizedBasename,
value
);
},
/**
* Finishes persisting data to disk and resets all state for this file.
*
* @return {Promise}
* @resolves When the object is finalized.
*/
_finalizeInternal() {
if (this._finalizePromise) {
// Finalization already in progress; return the pending promise. This is
// possible if `finalize` is called concurrently with shutdown.
return this._finalizePromise;
}
this._finalizePromise = (async () => {
await this._saver.finalize();
this._data = null;
this.dataReady = false;
})();
return this._finalizePromise;
},
/**
* Ensures that all data is persisted to disk, and prevents future calls to
* `saveSoon`. This is called automatically on shutdown, but can also be
* called explicitly when the file is no longer needed.
*/
async finalize() {
if (this._finalizePromise) {
throw new Error(`The file ${this.path} has already been finalized`);
}
// Wait for finalization before removing the shutdown blocker.
await this._finalizeInternal();
this._finalizeAt.removeBlocker(this._finalizeInternalBound);
},
};