Source code

Revision control

Other Tools

1
/* This Source Code Form is subject to the terms of the Mozilla Public
2
* License, v. 2.0. If a copy of the MPL was not distributed with this
3
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5
"use strict";
6
7
/**
8
* This file implements a mirror and two-way merger for synced bookmarks. The
9
* mirror matches the complete tree stored on the Sync server, and stages new
10
* bookmarks changed on the server since the last sync. The merger walks the
11
* local tree in Places and the mirrored remote tree, produces a new merged
12
* tree, then updates the local tree to reflect the merged tree.
13
*
14
* Let's start with an overview of the different classes, and how they fit
15
* together.
16
*
17
* - `SyncedBookmarksMirror` sets up the database, validates and upserts new
18
* incoming records, attaches to Places, and applies the changed records.
19
* During application, we fetch the local and remote bookmark trees, merge
20
* them, and update Places to match. Merging and application happen in a
21
* single transaction, so applying the merged tree won't collide with local
22
* changes. A failure at this point aborts the merge and leaves Places
23
* unchanged.
24
*
25
* - A `BookmarkTree` is a fully rooted tree that also notes deletions. A
26
* `BookmarkNode` represents a local item in Places, or a remote item in the
27
* mirror.
28
*
29
* - A `MergedBookmarkNode` holds a local node, a remote node, and a
30
* `MergeState` that indicates which node to prefer when updating Places and
31
* the server to match the merged tree.
32
*
33
* - `BookmarkObserverRecorder` records all changes made to Places during the
34
* merge, then dispatches `nsINavBookmarkObserver` notifications. Places uses
35
* these notifications to update the UI and internal caches. We can't dispatch
36
* during the merge because observers won't see the changes until the merge
37
* transaction commits and the database is consistent again.
38
*
39
* - After application, we flag all applied incoming items as merged, create
40
* Sync records for the locally new and updated items in Places, and upload
41
* the records to the server. At this point, all outgoing items are flagged as
42
* changed in Places, so the next sync can resume cleanly if the upload is
43
* interrupted or fails.
44
*
45
* - Once upload succeeds, we update the mirror with the uploaded records, so
46
* that the mirror matches the server again. An interruption or error here
47
* will leave the uploaded items flagged as changed in Places, so we'll merge
48
* them again on the next sync. This is redundant work, but shouldn't cause
49
* issues.
50
*/
51
52
var EXPORTED_SYMBOLS = ["SyncedBookmarksMirror"];
53
54
const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
55
const { XPCOMUtils } = ChromeUtils.import(
57
);
58
59
XPCOMUtils.defineLazyGlobalGetters(this, ["URL"]);
60
61
XPCOMUtils.defineLazyModuleGetters(this, {
67
});
68
69
XPCOMUtils.defineLazyGetter(this, "MirrorLog", () =>
70
Log.repository.getLogger("Sync.Engine.Bookmarks.Mirror")
71
);
72
73
const SyncedBookmarksMerger = Components.Constructor(
74
"@mozilla.org/browser/synced-bookmarks-merger;1",
75
"mozISyncedBookmarksMerger"
76
);
77
78
// These can be removed once they're exposed in a central location (bug
79
// 1375896).
80
const DB_URL_LENGTH_MAX = 65536;
81
const DB_TITLE_LENGTH_MAX = 4096;
82
83
// The current mirror database schema version. Bump for migrations, then add
84
// migration code to `migrateMirrorSchema`.
85
const MIRROR_SCHEMA_VERSION = 7;
86
87
const DEFAULT_MAX_FRECENCIES_TO_RECALCULATE = 400;
88
89
// Use a shared jankYielder in these functions
90
XPCOMUtils.defineLazyGetter(this, "yieldState", () => Async.yieldState());
91
92
/** Adapts a `Log.jsm` logger to a `mozISyncedBookmarksMirrorLogger`. */
93
class MirrorLoggerAdapter {
94
constructor(log) {
95
this.log = log;
96
}
97
98
get maxLevel() {
99
let level = this.log.level;
100
if (level <= Log.Level.All) {
101
return Ci.mozISyncedBookmarksMirrorLogger.LEVEL_TRACE;
102
}
103
if (level <= Log.Level.Info) {
104
return Ci.mozISyncedBookmarksMirrorLogger.LEVEL_DEBUG;
105
}
106
if (level <= Log.Level.Warn) {
107
return Ci.mozISyncedBookmarksMirrorLogger.LEVEL_WARN;
108
}
109
if (level <= Log.Level.Error) {
110
return Ci.mozISyncedBookmarksMirrorLogger.LEVEL_ERROR;
111
}
112
return Ci.mozISyncedBookmarksMirrorLogger.LEVEL_OFF;
113
}
114
115
trace(message) {
116
this.log.trace(message);
117
}
118
119
debug(message) {
120
this.log.debug(message);
121
}
122
123
warn(message) {
124
this.log.warn(message);
125
}
126
127
error(message) {
128
this.log.error(message);
129
}
130
}
131
132
/**
133
* A helper to track the progress of a merge for telemetry and shutdown hang
134
* reporting.
135
*/
136
class ProgressTracker {
137
constructor(recordStepTelemetry) {
138
this.recordStepTelemetry = recordStepTelemetry;
139
this.steps = [];
140
}
141
142
/**
143
* Records a merge step, updating the shutdown blocker state.
144
*
145
* @param {String} name A step name from `ProgressTracker.STEPS`. This is
146
* included in shutdown hang crash reports, along with the timestamp
147
* the step was recorded.
148
* @param {Number} [took] The time taken, in milliseconds.
149
* @param {Array} [counts] An array of additional counts to report in the
150
* shutdown blocker state.
151
*/
152
step(name, took = -1, counts = null) {
153
let info = { step: name, at: Date.now() };
154
if (took > -1) {
155
info.took = took;
156
}
157
if (counts) {
158
info.counts = counts;
159
}
160
this.steps.push(info);
161
}
162
163
/**
164
* Records a merge step with timings and counts for telemetry.
165
*
166
* @param {String} name The step name.
167
* @param {Number} took The time taken, in milliseconds.
168
* @param {Array} [counts] An array of additional `{ name, count }` tuples to
169
* record in telemetry for this step.
170
*/
171
stepWithTelemetry(name, took, counts = null) {
172
this.step(name, took, counts);
173
this.recordStepTelemetry(name, took, counts);
174
}
175
176
/**
177
* Records a merge step with the time taken and item count.
178
*
179
* @param {String} name The step name.
180
* @param {Number} took The time taken, in milliseconds.
181
* @param {Number} count The number of items handled in this step.
182
*/
183
stepWithItemCount(name, took, count) {
184
this.stepWithTelemetry(name, took, [{ name: "items", count }]);
185
}
186
187
/**
188
* Clears all recorded merge steps.
189
*/
190
reset() {
191
this.steps = [];
192
}
193
194
/**
195
* Returns the shutdown blocker state. This is included in shutdown hang
196
* crash reports, in the `AsyncShutdownTimeout` annotation.
197
*
198
* @see `fetchState` in `AsyncShutdown` for more details.
199
* @return {Object} A stringifiable object with the recorded steps.
200
*/
201
fetchState() {
202
return { steps: this.steps };
203
}
204
}
205
206
/** Merge steps for which we record progress. */
207
ProgressTracker.STEPS = {
208
FETCH_LOCAL_TREE: "fetchLocalTree",
209
FETCH_REMOTE_TREE: "fetchRemoteTree",
210
MERGE: "merge",
211
APPLY: "apply",
212
NOTIFY_OBSERVERS: "notifyObservers",
213
FETCH_LOCAL_CHANGE_RECORDS: "fetchLocalChangeRecords",
214
FINALIZE: "finalize",
215
};
216
217
/**
218
* A mirror maintains a copy of the complete tree as stored on the Sync server.
219
* It is persistent.
220
*
221
* The mirror schema is a hybrid of how Sync and Places represent bookmarks.
222
* The `items` table contains item attributes (title, kind, URL, etc.), while
223
* the `structure` table stores parent-child relationships and position.
224
* This is similar to how iOS encodes "value" and "structure" state,
225
* though we handle these differently when merging. See `BookmarkMerger` for
226
* details.
227
*
228
* There's no guarantee that the remote state is consistent. We might be missing
229
* parents or children, or a bookmark and its parent might disagree about where
230
* it belongs. This means we need a strategy to handle missing parents and
231
* children.
232
*
233
* We treat the `children` of the last parent we see as canonical, and ignore
234
* the child's `parentid` entirely. We also ignore missing children, and
235
* temporarily reparent bookmarks with missing parents to "unfiled". When we
236
* eventually see the missing items, either during a later sync or as part of
237
* repair, we'll fill in the mirror's gaps and fix up the local tree.
238
*
239
* During merging, we won't intentionally try to fix inconsistencies on the
240
* server, and opt to build as complete a tree as we can from the remote state,
241
* even if we diverge from what's in the mirror. See bug 1433512 for context.
242
*
243
* If a sync is interrupted, we resume downloading from the server collection
244
* last modified time, or the server last modified time of the most recent
245
* record if newer. New incoming records always replace existing records in the
246
* mirror.
247
*
248
* We delete the mirror database on client reset, including when the sync ID
249
* changes on the server, and when the user is node reassigned, disables the
250
* bookmarks engine, or signs out.
251
*/
252
class SyncedBookmarksMirror {
253
constructor(
254
db,
255
wasCorrupt = false,
256
{
257
recordStepTelemetry,
258
recordValidationTelemetry,
259
finalizeAt = PlacesUtils.history.shutdownClient.jsclient,
260
} = {}
261
) {
262
this.db = db;
263
this.wasCorrupt = wasCorrupt;
264
this.recordValidationTelemetry = recordValidationTelemetry;
265
266
this.merger = new SyncedBookmarksMerger();
267
this.merger.db = db.unsafeRawConnection.QueryInterface(
268
Ci.mozIStorageConnection
269
);
270
this.merger.logger = new MirrorLoggerAdapter(MirrorLog);
271
272
// Automatically close the database connection on shutdown. `progress`
273
// tracks state for shutdown hang reporting.
274
this.progress = new ProgressTracker(recordStepTelemetry);
275
this.finalizeController = new AbortController();
276
this.finalizeAt = finalizeAt;
277
this.finalizeBound = () => this.finalize({ alsoCleanup: false });
278
this.finalizeAt.addBlocker(
279
"SyncedBookmarksMirror: finalize",
280
this.finalizeBound,
281
{ fetchState: () => this.progress }
282
);
283
}
284
285
/**
286
* Sets up the mirror database connection and upgrades the mirror to the
287
* newest schema version. Automatically recreates the mirror if it's corrupt;
288
* throws on failure.
289
*
290
* @param {String} options.path
291
* The path to the mirror database file, either absolute or relative
292
* to the profile path.
293
* @param {Function} options.recordStepTelemetry
294
* A function with the signature `(name: String, took: Number,
295
* counts: Array?)`, where `name` is the name of the merge step,
296
* `took` is the time taken in milliseconds, and `counts` is an
297
* array of named counts (`{ name, count }` tuples) with additional
298
* counts for the step to record in the telemetry ping.
299
* @param {Function} options.recordValidationTelemetry
300
* A function with the signature `(took: Number, checked: Number,
301
* problems: Array)`, where `took` is the time taken to run
302
* validation in milliseconds, `checked` is the number of items
303
* checked, and `problems` is an array of named problem counts.
304
* @param {AsyncShutdown.Barrier} [options.finalizeAt]
305
* A shutdown phase, barrier, or barrier client that should
306
* automatically finalize the mirror when triggered. Exposed for
307
* testing.
308
* @return {SyncedBookmarksMirror}
309
* A mirror ready for use.
310
*/
311
static async open(options) {
312
let db = await PlacesUtils.promiseUnsafeWritableDBConnection();
313
if (!db) {
314
throw new TypeError("Can't open mirror without Places connection");
315
}
316
let path = OS.Path.join(OS.Constants.Path.profileDir, options.path);
317
let wasCorrupt = false;
318
try {
319
await attachAndInitMirrorDatabase(db, path);
320
} catch (ex) {
321
if (isDatabaseCorrupt(ex)) {
322
MirrorLog.warn(
323
"Error attaching mirror to Places; removing and " +
324
"recreating mirror",
325
ex
326
);
327
wasCorrupt = true;
328
await OS.File.remove(path);
329
await attachAndInitMirrorDatabase(db, path);
330
} else {
331
MirrorLog.error("Unrecoverable error attaching mirror to Places", ex);
332
throw ex;
333
}
334
}
335
return new SyncedBookmarksMirror(db, wasCorrupt, options);
336
}
337
338
/**
339
* Returns the newer of the bookmarks collection last modified time, or the
340
* server modified time of the newest record. The bookmarks engine uses this
341
* timestamp as the "high water mark" for all downloaded records. Each sync
342
* downloads and stores records that are strictly newer than this time.
343
*
344
* @return {Number}
345
* The high water mark time, in seconds.
346
*/
347
async getCollectionHighWaterMark() {
348
// The first case, where we have records with server modified times newer
349
// than the collection last modified time, occurs when a sync is interrupted
350
// before we call `setCollectionLastModified`. We subtract one second, the
351
// maximum time precision guaranteed by the server, so that we don't miss
352
// other records with the same time as the newest one we downloaded.
353
let rows = await this.db.executeCached(
354
`
355
SELECT MAX(
356
IFNULL((SELECT MAX(serverModified) - 1000 FROM items), 0),
357
IFNULL((SELECT CAST(value AS INTEGER) FROM meta
358
WHERE key = :modifiedKey), 0)
359
) AS highWaterMark`,
360
{ modifiedKey: SyncedBookmarksMirror.META_KEY.LAST_MODIFIED }
361
);
362
let highWaterMark = rows[0].getResultByName("highWaterMark");
363
return highWaterMark / 1000;
364
}
365
366
/**
367
* Updates the bookmarks collection last modified time. Note that this may
368
* be newer than the modified time of the most recent record.
369
*
370
* @param {Number|String} lastModifiedSeconds
371
* The collection last modified time, in seconds.
372
*/
373
async setCollectionLastModified(lastModifiedSeconds) {
374
let lastModified = Math.floor(lastModifiedSeconds * 1000);
375
if (!Number.isInteger(lastModified)) {
376
throw new TypeError("Invalid collection last modified time");
377
}
378
await this.db.executeBeforeShutdown(
379
"SyncedBookmarksMirror: setCollectionLastModified",
380
db =>
381
db.executeCached(
382
`
383
REPLACE INTO meta(key, value)
384
VALUES(:modifiedKey, :lastModified)`,
385
{
386
modifiedKey: SyncedBookmarksMirror.META_KEY.LAST_MODIFIED,
387
lastModified,
388
}
389
)
390
);
391
}
392
393
/**
394
* Returns the bookmarks collection sync ID. This corresponds to
395
* `PlacesSyncUtils.bookmarks.getSyncId`.
396
*
397
* @return {String}
398
* The sync ID, or `""` if one isn't set.
399
*/
400
async getSyncId() {
401
let rows = await this.db.executeCached(
402
`
403
SELECT value FROM meta WHERE key = :syncIdKey`,
404
{ syncIdKey: SyncedBookmarksMirror.META_KEY.SYNC_ID }
405
);
406
return rows.length ? rows[0].getResultByName("value") : "";
407
}
408
409
/**
410
* Ensures that the sync ID in the mirror is up-to-date with the server and
411
* Places, and discards the mirror on mismatch.
412
*
413
* The bookmarks engine store the same sync ID in Places and the mirror to
414
* "tie" the two together. This allows Sync to do the right thing if the
415
* database files are copied between profiles connected to different accounts.
416
*
417
* See `PlacesSyncUtils.bookmarks.ensureCurrentSyncId` for an explanation of
418
* how Places handles sync ID mismatches.
419
*
420
* @param {String} newSyncId
421
* The server's sync ID.
422
*/
423
async ensureCurrentSyncId(newSyncId) {
424
if (!newSyncId || typeof newSyncId != "string") {
425
throw new TypeError("Invalid new bookmarks sync ID");
426
}
427
let existingSyncId = await this.getSyncId();
428
if (existingSyncId == newSyncId) {
429
MirrorLog.trace("Sync ID up-to-date in mirror", { existingSyncId });
430
return;
431
}
432
MirrorLog.info(
433
"Sync ID changed from ${existingSyncId} to " +
434
"${newSyncId}; resetting mirror",
435
{ existingSyncId, newSyncId }
436
);
437
await this.db.executeBeforeShutdown(
438
"SyncedBookmarksMirror: ensureCurrentSyncId",
439
db =>
440
db.executeTransaction(async function() {
441
await resetMirror(db);
442
await db.execute(
443
`
444
REPLACE INTO meta(key, value)
445
VALUES(:syncIdKey, :newSyncId)`,
446
{ syncIdKey: SyncedBookmarksMirror.META_KEY.SYNC_ID, newSyncId }
447
);
448
})
449
);
450
}
451
452
/**
453
* Stores incoming or uploaded Sync records in the mirror. Rejects if any
454
* records are invalid.
455
*
456
* @param {PlacesItem[]} records
457
* Sync records to store in the mirror.
458
* @param {Boolean} [options.needsMerge]
459
* Indicates if the records were changed remotely since the last sync,
460
* and should be merged into the local tree. This option is set to
461
* `true` for incoming records, and `false` for successfully uploaded
462
* records. Tests can also pass `false` to set up an existing mirror.
463
* @param {AbortSignal} [options.signal]
464
* An abort signal that can be used to interrupt the operation. If
465
* omitted, storing incoming items can still be interrupted when the
466
* mirror is finalized.
467
*/
468
async store(records, { needsMerge = true, signal = null } = {}) {
469
let options = {
470
needsMerge,
471
signal: anyAborted(this.finalizeController.signal, signal),
472
};
473
await this.db.executeBeforeShutdown("SyncedBookmarksMirror: store", db =>
474
db.executeTransaction(async () => {
475
for (let record of records) {
476
if (options.signal.aborted) {
477
throw new SyncedBookmarksMirror.InterruptedError(
478
"Interrupted while storing incoming items"
479
);
480
}
481
let guid = PlacesSyncUtils.bookmarks.recordIdToGuid(record.id);
482
if (guid == PlacesUtils.bookmarks.rootGuid) {
483
// The engine should hard DELETE Places roots from the server.
484
throw new TypeError("Can't store Places root");
485
}
486
if (MirrorLog.level <= Log.Level.Trace) {
487
MirrorLog.trace(`Storing in mirror: ${record.cleartextToString()}`);
488
}
489
switch (record.type) {
490
case "bookmark":
491
await this.storeRemoteBookmark(record, options);
492
continue;
493
494
case "query":
495
await this.storeRemoteQuery(record, options);
496
continue;
497
498
case "folder":
499
await this.storeRemoteFolder(record, options);
500
continue;
501
502
case "livemark":
503
await this.storeRemoteLivemark(record, options);
504
continue;
505
506
case "separator":
507
await this.storeRemoteSeparator(record, options);
508
continue;
509
510
default:
511
if (record.deleted) {
512
await this.storeRemoteTombstone(record, options);
513
continue;
514
}
515
}
516
MirrorLog.warn("Ignoring record with unknown type", record.type);
517
}
518
})
519
);
520
}
521
522
/**
523
* Builds a complete merged tree from the local and remote trees, resolves
524
* value and structure conflicts, dedupes local items, applies the merged
525
* tree back to Places, and notifies observers about the changes.
526
*
527
* Merging and application happen in a transaction, meaning code that uses the
528
* main Places connection, including the UI, will fail to write to the
529
* database until the transaction commits. Asynchronous consumers will retry
530
* on `SQLITE_BUSY`; synchronous consumers will fail after waiting for 100ms.
531
* See bug 1305563, comment 122 for details.
532
*
533
* @param {Number} [options.localTimeSeconds]
534
* The current local time, in seconds.
535
* @param {Number} [options.remoteTimeSeconds]
536
* The current server time, in seconds.
537
* @param {String[]} [options.weakUpload]
538
* GUIDs of bookmarks to weakly upload.
539
* @param {Number} [options.maxFrecenciesToRecalculate]
540
* The maximum number of bookmark URL frecencies to recalculate after
541
* this merge. Frecency calculation blocks other Places writes, so we
542
* limit the number of URLs we process at once. We'll process either
543
* the next set of URLs after the next merge, or all remaining URLs
544
* when Places automatically fixes invalid frecencies on idle;
545
* whichever comes first.
546
* @param {Boolean} [options.notifyInStableOrder]
547
* If `true`, fire observer notifications for items in the same folder
548
* in a stable order. This is disabled by default, to avoid the cost
549
* of sorting the notifications, but enabled in some tests to simplify
550
* their checks.
551
* @param {AbortSignal} [options.signal]
552
* An abort signal that can be used to interrupt a merge when its
553
* associated `AbortController` is aborted. If omitted, the merge can
554
* still be interrupted when the mirror is finalized.
555
* @return {Object.<String, BookmarkChangeRecord>}
556
* A changeset containing locally changed and reconciled records to
557
* upload to the server, and to store in the mirror once upload
558
* succeeds.
559
*/
560
async apply({
561
localTimeSeconds,
562
remoteTimeSeconds,
563
weakUpload,
564
maxFrecenciesToRecalculate,
565
notifyInStableOrder,
566
signal = null,
567
} = {}) {
568
// We intentionally don't use `executeBeforeShutdown` in this function,
569
// since merging can take a while for large trees, and we don't want to
570
// block shutdown. Since all new items are in the mirror, we'll just try
571
// to merge again on the next sync.
572
573
let finalizeOrInterruptSignal = anyAborted(
574
this.finalizeController.signal,
575
signal
576
);
577
578
let changeRecords;
579
try {
580
changeRecords = await this.tryApply(
581
finalizeOrInterruptSignal,
582
localTimeSeconds,
583
remoteTimeSeconds,
584
weakUpload,
585
maxFrecenciesToRecalculate,
586
notifyInStableOrder
587
);
588
} finally {
589
this.progress.reset();
590
}
591
592
return changeRecords;
593
}
594
595
async tryApply(
596
signal,
597
localTimeSeconds,
598
remoteTimeSeconds,
599
weakUpload,
600
maxFrecenciesToRecalculate = DEFAULT_MAX_FRECENCIES_TO_RECALCULATE,
601
notifyInStableOrder = false
602
) {
603
let wasMerged = await withTiming("Merging bookmarks in Rust", () =>
604
this.merge(signal, localTimeSeconds, remoteTimeSeconds, weakUpload)
605
);
606
607
if (!wasMerged) {
608
MirrorLog.debug("No changes detected in both mirror and Places");
609
await updateFrecencies(this.db, maxFrecenciesToRecalculate);
610
return {};
611
}
612
613
// At this point, the database is consistent, so we can notify observers and
614
// inflate records for outgoing items.
615
616
let observersToNotify = new BookmarkObserverRecorder(this.db, {
617
maxFrecenciesToRecalculate,
618
signal,
619
notifyInStableOrder,
620
});
621
622
await withTiming(
623
"Notifying Places observers",
624
async () => {
625
try {
626
// Note that we don't use a transaction when fetching info for
627
// observers, so it's possible we might notify with stale info if the
628
// main connection changes Places between the time we finish merging,
629
// and the time we notify observers.
630
await observersToNotify.notifyAll();
631
} catch (ex) {
632
// Places relies on observer notifications to update internal caches.
633
// If notifying observers failed, these caches may be inconsistent,
634
// so we invalidate them just in case.
635
PlacesUtils.invalidateCachedGuids();
636
await PlacesUtils.keywords.invalidateCachedKeywords();
637
MirrorLog.warn("Error notifying Places observers", ex);
638
} finally {
639
await this.db.executeTransaction(async () => {
640
await this.db.execute(`DELETE FROM itemsAdded`);
641
await this.db.execute(`DELETE FROM guidsChanged`);
642
await this.db.execute(`DELETE FROM itemsChanged`);
643
await this.db.execute(`DELETE FROM itemsRemoved`);
644
await this.db.execute(`DELETE FROM itemsMoved`);
645
});
646
}
647
},
648
time =>
649
this.progress.stepWithTelemetry(
650
ProgressTracker.STEPS.NOTIFY_OBSERVERS,
651
time
652
)
653
);
654
655
let { changeRecords } = await withTiming(
656
"Fetching records for local items to upload",
657
async () => {
658
try {
659
let result = await this.fetchLocalChangeRecords(signal);
660
return result;
661
} finally {
662
await this.db.execute(`DELETE FROM itemsToUpload`);
663
}
664
},
665
(time, result) =>
666
this.progress.stepWithItemCount(
667
ProgressTracker.STEPS.FETCH_LOCAL_CHANGE_RECORDS,
668
time,
669
result.count
670
)
671
);
672
673
return changeRecords;
674
}
675
676
merge(
677
signal,
678
localTimeSeconds = Date.now() / 1000,
679
remoteTimeSeconds = 0,
680
weakUpload = []
681
) {
682
return new Promise((resolve, reject) => {
683
let op = null;
684
function onAbort() {
685
signal.removeEventListener("abort", onAbort);
686
op.cancel();
687
}
688
let callback = {
689
QueryInterface: ChromeUtils.generateQI([
690
Ci.mozISyncedBookmarksMirrorProgressListener,
691
Ci.mozISyncedBookmarksMirrorCallback,
692
]),
693
// `mozISyncedBookmarksMirrorProgressListener` methods.
694
onFetchLocalTree: (took, itemCount, deleteCount, problemsBag) => {
695
let counts = [
696
{
697
name: "items",
698
count: itemCount,
699
},
700
{
701
name: "deletions",
702
count: deleteCount,
703
},
704
];
705
this.progress.stepWithTelemetry(
706
ProgressTracker.STEPS.FETCH_LOCAL_TREE,
707
took,
708
counts
709
);
710
// We don't record local tree problems in validation telemetry.
711
},
712
onFetchRemoteTree: (took, itemCount, deleteCount, problemsBag) => {
713
let counts = [
714
{
715
name: "items",
716
count: itemCount,
717
},
718
{
719
name: "deletions",
720
count: deleteCount,
721
},
722
];
723
this.progress.stepWithTelemetry(
724
ProgressTracker.STEPS.FETCH_REMOTE_TREE,
725
took,
726
counts
727
);
728
// Record validation telemetry for problems in the remote tree.
729
let problems = bagToNamedCounts(problemsBag, [
730
"orphans",
731
"misparentedRoots",
732
"multipleParents",
733
"nonFolderParents",
734
"parentChildDisagreements",
735
"missingChildren",
736
]);
737
let checked = itemCount + deleteCount;
738
this.recordValidationTelemetry(took, checked, problems);
739
},
740
onMerge: (took, countsBag) => {
741
let counts = bagToNamedCounts(countsBag, [
742
"items",
743
"dupes",
744
"remoteRevives",
745
"localDeletes",
746
"localRevives",
747
"remoteDeletes",
748
]);
749
this.progress.stepWithTelemetry(
750
ProgressTracker.STEPS.MERGE,
751
took,
752
counts
753
);
754
},
755
onApply: took => {
756
this.progress.stepWithTelemetry(ProgressTracker.STEPS.APPLY, took);
757
},
758
// `mozISyncedBookmarksMirrorCallback` methods.
759
handleSuccess(result) {
760
signal.removeEventListener("abort", onAbort);
761
resolve(result);
762
},
763
handleError(code, message) {
764
signal.removeEventListener("abort", onAbort);
765
switch (code) {
766
case Cr.NS_ERROR_STORAGE_BUSY:
767
reject(
768
new SyncedBookmarksMirror.MergeConflictError(
769
"Local tree changed during merge"
770
)
771
);
772
break;
773
774
case Cr.NS_ERROR_ABORT:
775
reject(new SyncedBookmarksMirror.InterruptedError(message));
776
break;
777
778
default:
779
reject(new SyncedBookmarksMirror.MergeError(message));
780
}
781
},
782
};
783
op = this.merger.merge(
784
localTimeSeconds,
785
remoteTimeSeconds,
786
weakUpload,
787
callback
788
);
789
if (signal.aborted) {
790
op.cancel();
791
} else {
792
signal.addEventListener("abort", onAbort);
793
}
794
});
795
}
796
797
/**
798
* Discards the mirror contents. This is called when the user is node
799
* reassigned, disables the bookmarks engine, or signs out.
800
*/
801
async reset() {
802
await this.db.executeBeforeShutdown("SyncedBookmarksMirror: reset", db =>
803
db.executeTransaction(() => resetMirror(db))
804
);
805
}
806
807
/**
808
* Fetches the GUIDs of all items in the remote tree that need to be merged
809
* into the local tree.
810
*
811
* @return {String[]}
812
* Remotely changed GUIDs that need to be merged into Places.
813
*/
814
async fetchUnmergedGuids() {
815
let rows = await this.db.execute(`
816
SELECT guid FROM items
817
WHERE needsMerge
818
ORDER BY guid`);
819
return rows.map(row => row.getResultByName("guid"));
820
}
821
822
async storeRemoteBookmark(record, { needsMerge, signal }) {
823
let guid = PlacesSyncUtils.bookmarks.recordIdToGuid(record.id);
824
825
let url = validateURL(record.bmkUri);
826
if (url) {
827
await this.maybeStoreRemoteURL(url);
828
}
829
830
let parentGuid = PlacesSyncUtils.bookmarks.recordIdToGuid(record.parentid);
831
let serverModified = determineServerModified(record);
832
let dateAdded = determineDateAdded(record);
833
let title = validateTitle(record.title);
834
let keyword = validateKeyword(record.keyword);
835
let validity = url
836
? Ci.mozISyncedBookmarksMerger.VALIDITY_VALID
837
: Ci.mozISyncedBookmarksMerger.VALIDITY_REPLACE;
838
839
await this.db.executeCached(
840
`
841
REPLACE INTO items(guid, parentGuid, serverModified, needsMerge, kind,
842
dateAdded, title, keyword, validity,
843
urlId)
844
VALUES(:guid, :parentGuid, :serverModified, :needsMerge, :kind,
845
:dateAdded, NULLIF(:title, ''), :keyword, :validity,
846
(SELECT id FROM urls
847
WHERE hash = hash(:url) AND
848
url = :url))`,
849
{
850
guid,
851
parentGuid,
852
serverModified,
853
needsMerge,
854
kind: Ci.mozISyncedBookmarksMerger.KIND_BOOKMARK,
855
dateAdded,
856
title,
857
keyword,
858
url: url ? url.href : null,
859
validity,
860
}
861
);
862
863
let tags = record.tags;
864
if (tags && Array.isArray(tags)) {
865
for (let rawTag of tags) {
866
if (signal.aborted) {
867
throw new SyncedBookmarksMirror.InterruptedError(
868
"Interrupted while storing tags for incoming bookmark"
869
);
870
}
871
let tag = validateTag(rawTag);
872
if (!tag) {
873
continue;
874
}
875
await this.db.executeCached(
876
`
877
INSERT INTO tags(itemId, tag)
878
SELECT id, :tag FROM items
879
WHERE guid = :guid`,
880
{ tag, guid }
881
);
882
}
883
}
884
}
885
886
async storeRemoteQuery(record, { needsMerge }) {
887
let guid = PlacesSyncUtils.bookmarks.recordIdToGuid(record.id);
888
889
let validity = Ci.mozISyncedBookmarksMerger.VALIDITY_VALID;
890
891
let url = validateURL(record.bmkUri);
892
if (url) {
893
// The query has a valid URL. Determine if we need to rewrite and reupload
894
// it.
895
let params = new URLSearchParams(url.href.slice(url.protocol.length));
896
let type = +params.get("type");
897
if (type == Ci.nsINavHistoryQueryOptions.RESULTS_AS_TAG_CONTENTS) {
898
// Legacy tag queries with this type use a `place:` URL with a `folder`
899
// param that points to the tag folder ID. Rewrite the query to directly
900
// reference the tag stored in its `folderName`, then flag the rewritten
901
// query for reupload.
902
let tagFolderName = validateTag(record.folderName);
903
if (tagFolderName) {
904
try {
905
url.href = `place:tag=${tagFolderName}`;
906
validity = Ci.mozISyncedBookmarksMerger.VALIDITY_REUPLOAD;
907
} catch (ex) {
908
// The tag folder name isn't URL-encoded (bug 1449939), so we might
909
// produce an invalid URL. However, invalid URLs are already likely
910
// to cause other issues, and it's better to replace or delete the
911
// query than break syncing or the Firefox UI.
912
url = null;
913
}
914
} else {
915
// The tag folder name is invalid, so replace or delete the remote
916
// copy.
917
url = null;
918
}
919
} else {
920
let folder = params.get("folder");
921
if (folder && !params.has("excludeItems")) {
922
// We don't sync enough information to rewrite other queries with a
923
// `folder` param (bug 1377175). Referencing a nonexistent folder ID
924
// causes the query to return all items in the database, so we add
925
// `excludeItems=1` to stop it from doing so. We also flag the
926
// rewritten query for reupload.
927
try {
928
url.href = `${url.href}&excludeItems=1`;
929
validity = Ci.mozISyncedBookmarksMerger.VALIDITY_REUPLOAD;
930
} catch (ex) {
931
url = null;
932
}
933
}
934
}
935
936
// Other queries are implicitly valid, and don't need to be reuploaded
937
// or replaced.
938
}
939
940
if (url) {
941
await this.maybeStoreRemoteURL(url);
942
} else {
943
// If the query doesn't have a valid URL, we must replace the remote copy
944
// with either a valid local copy, or a tombstone if the query doesn't
945
// exist locally.
946
validity = Ci.mozISyncedBookmarksMerger.VALIDITY_REPLACE;
947
}
948
949
let parentGuid = PlacesSyncUtils.bookmarks.recordIdToGuid(record.parentid);
950
let serverModified = determineServerModified(record);
951
let dateAdded = determineDateAdded(record);
952
let title = validateTitle(record.title);
953
954
await this.db.executeCached(
955
`
956
REPLACE INTO items(guid, parentGuid, serverModified, needsMerge, kind,
957
dateAdded, title,
958
urlId,
959
validity)
960
VALUES(:guid, :parentGuid, :serverModified, :needsMerge, :kind,
961
:dateAdded, NULLIF(:title, ''),
962
(SELECT id FROM urls
963
WHERE hash = hash(:url) AND
964
url = :url),
965
:validity)`,
966
{
967
guid,
968
parentGuid,
969
serverModified,
970
needsMerge,
971
kind: Ci.mozISyncedBookmarksMerger.KIND_QUERY,
972
dateAdded,
973
title,
974
url: url ? url.href : null,
975
validity,
976
}
977
);
978
}
979
980
async storeRemoteFolder(record, { needsMerge, signal }) {
981
let guid = PlacesSyncUtils.bookmarks.recordIdToGuid(record.id);
982
let parentGuid = PlacesSyncUtils.bookmarks.recordIdToGuid(record.parentid);
983
let serverModified = determineServerModified(record);
984
let dateAdded = determineDateAdded(record);
985
let title = validateTitle(record.title);
986
987
await this.db.executeCached(
988
`
989
REPLACE INTO items(guid, parentGuid, serverModified, needsMerge, kind,
990
dateAdded, title)
991
VALUES(:guid, :parentGuid, :serverModified, :needsMerge, :kind,
992
:dateAdded, NULLIF(:title, ''))`,
993
{
994
guid,
995
parentGuid,
996
serverModified,
997
needsMerge,
998
kind: Ci.mozISyncedBookmarksMerger.KIND_FOLDER,
999
dateAdded,
1000
title,
1001
}
1002
);
1003
1004
let children = record.children;
1005
if (children && Array.isArray(children)) {
1006
let offset = 0;
1007
for (let chunk of PlacesUtils.chunkArray(
1008
children,
1009
this.db.variableLimit - 1
1010
)) {
1011
if (signal.aborted) {
1012
throw new SyncedBookmarksMirror.InterruptedError(
1013
"Interrupted while storing children for incoming folder"
1014
);
1015
}
1016
// Builds a fragment like `(?2, ?1, 0), (?3, ?1, 1), ...`, where ?1 is
1017
// the folder's GUID, [?2, ?3] are the first and second child GUIDs
1018
// (SQLite binding parameters index from 1), and [0, 1] are the
1019
// positions. This lets us store the folder's children using as few
1020
// statements as possible.
1021
let valuesFragment = Array.from(
1022
{ length: chunk.length },
1023
(_, index) => `(?${index + 2}, ?1, ${offset + index})`
1024
).join(",");
1025
await this.db.execute(
1026
`
1027
INSERT INTO structure(guid, parentGuid, position)
1028
VALUES ${valuesFragment}`,
1029
[guid, ...chunk.map(PlacesSyncUtils.bookmarks.recordIdToGuid)]
1030
);
1031
offset += chunk.length;
1032
}
1033
}
1034
}
1035
1036
async storeRemoteLivemark(record, { needsMerge }) {
1037
let guid = PlacesSyncUtils.bookmarks.recordIdToGuid(record.id);
1038
let parentGuid = PlacesSyncUtils.bookmarks.recordIdToGuid(record.parentid);
1039
let serverModified = determineServerModified(record);
1040
let feedURL = validateURL(record.feedUri);
1041
let dateAdded = determineDateAdded(record);
1042
let title = validateTitle(record.title);
1043
let siteURL = validateURL(record.siteUri);
1044
1045
let validity = feedURL
1046
? Ci.mozISyncedBookmarksMerger.VALIDITY_VALID
1047
: Ci.mozISyncedBookmarksMerger.VALIDITY_REPLACE;
1048
1049
await this.db.executeCached(
1050
`
1051
REPLACE INTO items(guid, parentGuid, serverModified, needsMerge, kind,
1052
dateAdded, title, feedURL, siteURL, validity)
1053
VALUES(:guid, :parentGuid, :serverModified, :needsMerge, :kind,
1054
:dateAdded, NULLIF(:title, ''), :feedURL, :siteURL, :validity)`,
1055
{
1056
guid,
1057
parentGuid,
1058
serverModified,
1059
needsMerge,
1060
kind: Ci.mozISyncedBookmarksMerger.KIND_LIVEMARK,
1061
dateAdded,
1062
title,
1063
feedURL: feedURL ? feedURL.href : null,
1064
siteURL: siteURL ? siteURL.href : null,
1065
validity,
1066
}
1067
);
1068
}
1069
1070
async storeRemoteSeparator(record, { needsMerge }) {
1071
let guid = PlacesSyncUtils.bookmarks.recordIdToGuid(record.id);
1072
let parentGuid = PlacesSyncUtils.bookmarks.recordIdToGuid(record.parentid);
1073
let serverModified = determineServerModified(record);
1074
let dateAdded = determineDateAdded(record);
1075
1076
await this.db.executeCached(
1077
`
1078
REPLACE INTO items(guid, parentGuid, serverModified, needsMerge, kind,
1079
dateAdded)
1080
VALUES(:guid, :parentGuid, :serverModified, :needsMerge, :kind,
1081
:dateAdded)`,
1082
{
1083
guid,
1084
parentGuid,
1085
serverModified,
1086
needsMerge,
1087
kind: Ci.mozISyncedBookmarksMerger.KIND_SEPARATOR,
1088
dateAdded,
1089
}
1090
);
1091
}
1092
1093
async storeRemoteTombstone(record, { needsMerge }) {
1094
let guid = PlacesSyncUtils.bookmarks.recordIdToGuid(record.id);
1095
let serverModified = determineServerModified(record);
1096
1097
await this.db.executeCached(
1098
`
1099
REPLACE INTO items(guid, serverModified, needsMerge, isDeleted)
1100
VALUES(:guid, :serverModified, :needsMerge, 1)`,
1101
{ guid, serverModified, needsMerge }
1102
);
1103
}
1104
1105
async maybeStoreRemoteURL(url) {
1106
await this.db.executeCached(
1107
`
1108
INSERT OR IGNORE INTO urls(guid, url, hash, revHost)
1109
VALUES(IFNULL((SELECT guid FROM urls
1110
WHERE hash = hash(:url) AND
1111
url = :url),
1112
GENERATE_GUID()), :url, hash(:url), :revHost)`,
1113
{ url: url.href, revHost: PlacesUtils.getReversedHost(url) }
1114
);
1115
}
1116
1117
/**
1118
* Inflates Sync records for all staged outgoing items.
1119
*
1120
* @param {AbortSignal} signal
1121
* Stops fetching records when the associated `AbortController`
1122
* is aborted.
1123
* @return {Object}
1124
* A `{ changeRecords, count }` tuple, where `changeRecords` is a
1125
* changeset containing Sync record cleartexts for outgoing items and
1126
* tombstones, keyed by their Sync record IDs, and `count` is the
1127
* number of records.
1128
*/
1129
async fetchLocalChangeRecords(signal) {
1130
let changeRecords = {};
1131
let childRecordIdsByLocalParentId = new Map();
1132
let tagsByLocalId = new Map();
1133
1134
let childGuidRows = [];
1135
await this.db.execute(
1136
`SELECT parentId, guid FROM structureToUpload
1137
ORDER BY parentId, position`,
1138
null,
1139
(row, cancel) => {
1140
if (signal.aborted) {
1141
cancel();
1142
} else {
1143
// `Sqlite.jsm` callbacks swallow exceptions (bug 1387775), so we
1144
// accumulate all rows in an array, and process them after.
1145
childGuidRows.push(row);
1146
}
1147
}
1148
);
1149
1150
await Async.yieldingForEach(
1151
childGuidRows,
1152
row => {
1153
if (signal.aborted) {
1154
throw new SyncedBookmarksMirror.InterruptedError(
1155
"Interrupted while fetching structure to upload"
1156
);
1157
}
1158
let localParentId = row.getResultByName("parentId");
1159
let childRecordId = PlacesSyncUtils.bookmarks.guidToRecordId(
1160
row.getResultByName("guid")
1161
);
1162
let childRecordIds = childRecordIdsByLocalParentId.get(localParentId);
1163
if (childRecordIds) {
1164
childRecordIds.push(childRecordId);
1165
} else {
1166
childRecordIdsByLocalParentId.set(localParentId, [childRecordId]);
1167
}
1168
},
1169
yieldState
1170
);
1171
1172
let tagRows = [];
1173
await this.db.execute(
1174
`SELECT id, tag FROM tagsToUpload`,
1175
null,
1176
(row, cancel) => {
1177
if (signal.aborted) {
1178
cancel();
1179
} else {
1180
tagRows.push(row);
1181
}
1182
}
1183
);
1184
1185
await Async.yieldingForEach(
1186
tagRows,
1187
row => {
1188
if (signal.aborted) {
1189
throw new SyncedBookmarksMirror.InterruptedError(
1190
"Interrupted while fetching tags to upload"
1191
);
1192
}
1193
let localId = row.getResultByName("id");
1194
let tag = row.getResultByName("tag");
1195
let tags = tagsByLocalId.get(localId);
1196
if (tags) {
1197
tags.push(tag);
1198
} else {
1199
tagsByLocalId.set(localId, [tag]);
1200
}
1201
},
1202
yieldState
1203
);
1204
1205
let itemRows = [];
1206
await this.db.execute(
1207
`SELECT id, syncChangeCounter, guid, isDeleted, type, isQuery,
1208
tagFolderName, keyword, url, IFNULL(title, '') AS title,
1209
position, parentGuid,
1210
IFNULL(parentTitle, '') AS parentTitle, dateAdded
1211
FROM itemsToUpload`,
1212
null,
1213
(row, cancel) => {
1214
if (signal.interrupted) {
1215
cancel();
1216
} else {
1217
itemRows.push(row);
1218
}
1219
}
1220
);
1221
1222
await Async.yieldingForEach(
1223
itemRows,
1224
row => {
1225
if (signal.aborted) {
1226
throw new SyncedBookmarksMirror.InterruptedError(
1227
"Interrupted while fetching items to upload"
1228
);
1229
}
1230
let syncChangeCounter = row.getResultByName("syncChangeCounter");
1231
1232
let guid = row.getResultByName("guid");
1233
let recordId = PlacesSyncUtils.bookmarks.guidToRecordId(guid);
1234
1235
// Tombstones don't carry additional properties.
1236
let isDeleted = row.getResultByName("isDeleted");
1237
if (isDeleted) {
1238
changeRecords[recordId] = new BookmarkChangeRecord(
1239
syncChangeCounter,
1240
{
1241
id: recordId,
1242
deleted: true,
1243
}
1244
);
1245
return;
1246
}
1247
1248
let parentGuid = row.getResultByName("parentGuid");
1249
let parentRecordId = PlacesSyncUtils.bookmarks.guidToRecordId(
1250
parentGuid
1251
);
1252
1253
let type = row.getResultByName("type");
1254
switch (type) {
1255
case PlacesUtils.bookmarks.TYPE_BOOKMARK: {
1256
let isQuery = row.getResultByName("isQuery");
1257
if (isQuery) {
1258
let queryCleartext = {
1259
id: recordId,
1260
type: "query",
1261
// We ignore `parentid` and use the parent's `children`, but older
1262
// Desktops and Android use `parentid` as the canonical parent.
1263
// iOS is stricter and requires both `children` and `parentid` to
1264
// match.
1265
parentid: parentRecordId,
1266
// Older Desktops use `hasDupe` (along with `parentName` for
1267
// deduping), if hasDupe is true, then they won't attempt deduping
1268
// (since they believe that a duplicate for this record should
1269
// exist). We set it to true to prevent them from applying their
1270
// deduping logic.
1271
hasDupe: true,
1272
parentName: row.getResultByName("parentTitle"),
1273
// Omit `dateAdded` from the record if it's not set locally.
1274
dateAdded: row.getResultByName("dateAdded") || undefined,
1275
bmkUri: row.getResultByName("url"),
1276
title: row.getResultByName("title"),
1277
// folderName should never be an empty string or null
1278
folderName: row.getResultByName("tagFolderName") || undefined,
1279
};
1280
changeRecords[recordId] = new BookmarkChangeRecord(
1281
syncChangeCounter,
1282
queryCleartext
1283
);
1284
return;
1285
}
1286
1287
let bookmarkCleartext = {
1288
id: recordId,
1289
type: "bookmark",
1290
parentid: parentRecordId,
1291
hasDupe: true,
1292
parentName: row.getResultByName("parentTitle"),
1293
dateAdded: row.getResultByName("dateAdded") || undefined,
1294
bmkUri: row.getResultByName("url"),
1295
title: row.getResultByName("title"),
1296
};
1297
let keyword = row.getResultByName("keyword");
1298
if (keyword) {
1299
bookmarkCleartext.keyword = keyword;
1300
}
1301
let localId = row.getResultByName("id");
1302
let tags = tagsByLocalId.get(localId);
1303
if (tags) {
1304
bookmarkCleartext.tags = tags;
1305
}
1306
changeRecords[recordId] = new BookmarkChangeRecord(
1307
syncChangeCounter,
1308
bookmarkCleartext
1309
);
1310
return;
1311
}
1312
1313
case PlacesUtils.bookmarks.TYPE_FOLDER: {
1314
let folderCleartext = {
1315
id: recordId,
1316
type: "folder",
1317
parentid: parentRecordId,
1318
hasDupe: true,
1319
parentName: row.getResultByName("parentTitle"),
1320
dateAdded: row.getResultByName("dateAdded") || undefined,
1321
title: row.getResultByName("title"),
1322
};
1323
let localId = row.getResultByName("id");
1324
let childRecordIds = childRecordIdsByLocalParentId.get(localId);
1325
folderCleartext.children = childRecordIds || [];
1326
changeRecords[recordId] = new BookmarkChangeRecord(
1327
syncChangeCounter,
1328
folderCleartext
1329
);
1330
return;
1331
}
1332
1333
case PlacesUtils.bookmarks.TYPE_SEPARATOR: {
1334
let separatorCleartext = {
1335
id: recordId,
1336
type: "separator",
1337
parentid: parentRecordId,
1338
hasDupe: true,
1339
parentName: row.getResultByName("parentTitle"),
1340
dateAdded: row.getResultByName("dateAdded") || undefined,
1341
// Older Desktops use `pos` for deduping.
1342
pos: row.getResultByName("position"),
1343
};
1344
changeRecords[recordId] = new BookmarkChangeRecord(
1345
syncChangeCounter,
1346
separatorCleartext
1347
);
1348
return;
1349
}
1350
1351
default:
1352
throw new TypeError("Can't create record for unknown Places item");
1353
}
1354
},
1355
yieldState
1356
);
1357
1358
return { changeRecords, count: itemRows.length };
1359
}
1360
1361
/**
1362
* Closes the mirror database connection. This is called automatically on
1363
* shutdown, but may also be called explicitly when the mirror is no longer
1364
* needed.
1365
*
1366
* @param {Boolean} [options.alsoCleanup]
1367
* If specified, drop all temp tables, views, and triggers,
1368
* and detach from the mirror database before closing the
1369
* connection. Defaults to `true`.
1370
*/
1371
finalize({ alsoCleanup = true } = {}) {
1372
if (!this.finalizePromise) {
1373
this.finalizePromise = (async () => {
1374
this.progress.step(ProgressTracker.STEPS.FINALIZE);
1375
this.finalizeController.abort();
1376
this.merger.reset();
1377
if (alsoCleanup) {
1378
// If the mirror is finalized explicitly, clean up temp entities and
1379
// detach from the mirror database. We can skip this for automatic
1380
// finalization, since the Places connection is already shutting
1381
// down.
1382
await cleanupMirrorDatabase(this.db);
1383
}
1384
await this.db.execute(`PRAGMA mirror.optimize(0x02)`);
1385
await this.db.execute(`DETACH mirror`);
1386
this.finalizeAt.removeBlocker(this.finalizeBound);
1387
})();
1388
}
1389
return this.finalizePromise;
1390
}
1391
}
1392
1393
this.SyncedBookmarksMirror = SyncedBookmarksMirror;
1394
1395
/** Key names for the key-value `meta` table. */
1396
SyncedBookmarksMirror.META_KEY = {
1397
LAST_MODIFIED: "collection/lastModified",
1398
SYNC_ID: "collection/syncId",
1399
};
1400
1401
/**
1402
* An error thrown when the merge was interrupted.
1403
*/
1404
class InterruptedError extends Error {
1405
constructor(message) {
1406
super(message);
1407
this.name = "InterruptedError";
1408
}
1409
}
1410
SyncedBookmarksMirror.InterruptedError = InterruptedError;
1411
1412
/**
1413
* An error thrown when the merge failed for an unexpected reason.
1414
*/
1415
class MergeError extends Error {
1416
constructor(message) {
1417
super(message);
1418
this.name = "MergeError";
1419
}
1420
}
1421
SyncedBookmarksMirror.MergeError = MergeError;
1422
1423
/**
1424
* An error thrown when the merge can't proceed because the local tree
1425
* changed during the merge.
1426
*/
1427
class MergeConflictError extends Error {
1428
constructor(message) {
1429
super(message);
1430
this.name = "MergeConflictError";
1431
}
1432
}
1433
SyncedBookmarksMirror.MergeConflictError = MergeConflictError;
1434
1435
/**
1436
* An error thrown when the mirror database is corrupt, or can't be migrated to
1437
* the latest schema version, and must be replaced.
1438
*/
1439
class DatabaseCorruptError extends Error {
1440
constructor(message) {
1441
super(message);
1442
this.name = "DatabaseCorruptError";
1443
}
1444
}
1445
1446
// Indicates if the mirror should be replaced because the database file is
1447
// corrupt.
1448
function isDatabaseCorrupt(error) {
1449
if (error instanceof DatabaseCorruptError) {
1450
return true;
1451
}
1452
if (error.errors) {
1453
return error.errors.some(
1454
error =>
1455
error instanceof Ci.mozIStorageError &&
1456
(error.result == Ci.mozIStorageError.CORRUPT ||
1457
error.result == Ci.mozIStorageError.NOTADB)
1458
);
1459
}
1460
return false;
1461
}
1462
1463
/**
1464
* Attaches a cloned Places database connection to the mirror database,
1465
* migrates the mirror schema to the latest version, and creates temporary
1466
* tables, views, and triggers.
1467
*
1468
* @param {Sqlite.OpenedConnection} db
1469
* The Places database connection.
1470
* @param {String} path
1471
* The full path to the mirror database file.
1472
*/
1473
async function attachAndInitMirrorDatabase(db, path) {
1474
await db.execute(`ATTACH :path AS mirror`, { path });
1475
try {
1476
await db.executeTransaction(async function() {
1477
let currentSchemaVersion = await db.getSchemaVersion("mirror");
1478
if (currentSchemaVersion > 0) {
1479
if (currentSchemaVersion < MIRROR_SCHEMA_VERSION) {
1480
await migrateMirrorSchema(db, currentSchemaVersion);
1481
}
1482
} else {
1483
await initializeMirrorDatabase(db);
1484
}
1485
// Downgrading from a newer profile to an older profile rolls back the
1486
// schema version, but leaves all new columns in place. We'll run the
1487
// migration logic again on the next upgrade.
1488
await db.setSchemaVersion(MIRROR_SCHEMA_VERSION, "mirror");
1489
await initializeTempMirrorEntities(db);
1490
});
1491
} catch (ex) {
1492
await db.execute(`DETACH mirror`);
1493
throw ex;
1494
}
1495
}
1496
1497
/**
1498
* Migrates the mirror database schema to the latest version.
1499
*
1500
* @param {Sqlite.OpenedConnection} db
1501
* The mirror database connection.
1502
* @param {Number} currentSchemaVersion
1503
* The current mirror database schema version.
1504
*/
1505
async function migrateMirrorSchema(db, currentSchemaVersion) {
1506
if (currentSchemaVersion < 5) {
1507
// The mirror was pref'd off by default for schema versions 1-4.
1508
throw new DatabaseCorruptError(
1509
`Can't migrate from schema version ${currentSchemaVersion}; too old`
1510
);
1511
}
1512
if (currentSchemaVersion < 6) {
1513
await db.execute(`CREATE INDEX IF NOT EXISTS mirror.itemURLs ON
1514
items(urlId)`);
1515
await db.execute(`CREATE INDEX IF NOT EXISTS mirror.itemKeywords ON
1516
items(keyword) WHERE keyword NOT NULL`);
1517
}
1518
if (currentSchemaVersion < 7) {
1519
await db.execute(`CREATE INDEX IF NOT EXISTS mirror.structurePositions ON
1520
structure(parentGuid, position)`);
1521
}
1522
}
1523
1524
/**
1525
* Initializes a new mirror database, creating persistent tables, indexes, and
1526
* roots.
1527
*
1528
* @param {Sqlite.OpenedConnection} db
1529
* The mirror database connection.
1530
*/
1531
async function initializeMirrorDatabase(db) {
1532
// Key-value metadata table. Stores the server collection last modified time
1533
// and sync ID.
1534
await db.execute(`CREATE TABLE mirror.meta(
1535
key TEXT PRIMARY KEY,
1536
value NOT NULL
1537
) WITHOUT ROWID`);
1538
1539
// Note: description and loadInSidebar are not used as of Firefox 63, but
1540
// remain to avoid rebuilding the database if the user happens to downgrade.
1541
await db.execute(`CREATE TABLE mirror.items(
1542
id INTEGER PRIMARY KEY,
1543
guid TEXT UNIQUE NOT NULL,
1544
/* The "parentid" from the record. */
1545
parentGuid TEXT,
1546
/* The server modified time, in milliseconds. */
1547
serverModified INTEGER NOT NULL DEFAULT 0,
1548
needsMerge BOOLEAN NOT NULL DEFAULT 0,
1549
validity INTEGER NOT NULL DEFAULT ${
1550
Ci.mozISyncedBookmarksMerger.VALIDITY_VALID
1551
},
1552
isDeleted BOOLEAN NOT NULL DEFAULT 0,
1553
kind INTEGER NOT NULL DEFAULT -1,
1554
/* The creation date, in milliseconds. */
1555
dateAdded INTEGER NOT NULL DEFAULT 0,
1556
title TEXT,
1557
urlId INTEGER REFERENCES urls(id)
1558
ON DELETE SET NULL,
1559
keyword TEXT,
1560
description TEXT,
1561
loadInSidebar BOOLEAN,
1562
smartBookmarkName TEXT,
1563
feedURL TEXT,
1564
siteURL TEXT
1565
)`);
1566
1567
await db.execute(`CREATE TABLE mirror.structure(
1568
guid TEXT,
1569
parentGuid TEXT REFERENCES items(guid)
1570
ON DELETE CASCADE,
1571
position INTEGER NOT NULL,
1572
PRIMARY KEY(parentGuid, guid)
1573
) WITHOUT ROWID`);
1574
1575
await db.execute(`CREATE TABLE mirror.urls(
1576
id INTEGER PRIMARY KEY,
1577
guid TEXT NOT NULL,
1578
url TEXT NOT NULL,
1579
hash INTEGER NOT NULL,
1580
revHost TEXT NOT NULL
1581
)`);
1582
1583
await db.execute(`CREATE TABLE mirror.tags(
1584
itemId INTEGER NOT NULL REFERENCES items(id)
1585
ON DELETE CASCADE,
1586
tag TEXT NOT NULL
1587
)`);
1588
1589
await db.execute(
1590
`CREATE INDEX mirror.structurePositions ON structure(parentGuid, position)`
1591
);
1592
1593
await db.execute(`CREATE INDEX mirror.urlHashes ON urls(hash)`);
1594
1595
await db.execute(`CREATE INDEX mirror.itemURLs ON items(urlId)`);
1596
1597
await db.execute(`CREATE INDEX mirror.itemKeywords ON items(keyword)
1598
WHERE keyword NOT NULL`);
1599
1600
await createMirrorRoots(db);
1601
}
1602
1603
/**
1604
* Drops all temp tables, views, and triggers used for merging, and detaches
1605
* from the mirror database.
1606
*
1607
* @param {Sqlite.OpenedConnection} db
1608
* The mirror database connection.
1609
*/
1610
async function cleanupMirrorDatabase(db) {
1611
await db.executeTransaction(async function() {
1612
await db.execute(`DROP TABLE changeGuidOps`);
1613
await db.execute(`DROP TABLE itemsToApply`);
1614
await db.execute(`DROP TABLE applyNewLocalStructureOps`);
1615
await db.execute(`DROP VIEW localTags`);
1616
await db.execute(`DROP TABLE itemsAdded`);
1617
await db.execute(`DROP TABLE guidsChanged`);
1618
await db.execute(`DROP TABLE itemsChanged`);
1619
await db.execute(`DROP TABLE itemsMoved`);
1620
await db.execute(`DROP TABLE itemsRemoved`);
1621
await db.execute(`DROP TABLE itemsToUpload`);
1622
await db.execute(`DROP TABLE structureToUpload`);
1623
await db.execute(`DROP TABLE tagsToUpload`);
1624
});
1625
}
1626
1627
/**
1628
* Sets up the syncable roots. All items in the mirror we apply will descend
1629
* from these roots - however, malformed records from the server which create
1630
* a different root *will* be created in the mirror - just not applied.
1631
*
1632
*
1633
* @param {Sqlite.OpenedConnection} db
1634
* The mirror database connection.
1635
*/
1636
async function createMirrorRoots(db) {
1637
const syncableRoots = [
1638
{
1639
guid: PlacesUtils.bookmarks.rootGuid,
1640
// The Places root is its own parent, to satisfy the foreign key and
1641
// `NOT NULL` constraints on `structure`.
1642
parentGuid: PlacesUtils.bookmarks.rootGuid,
1643
position: -1,
1644
needsMerge: false,
1645
},
1646
...PlacesUtils.bookmarks.userContentRoots.map((guid, position) => {
1647
return {
1648
guid,
1649
parentGuid: PlacesUtils.bookmarks.rootGuid,
1650
position,
1651
needsMerge: true,
1652
};
1653
}),
1654
];
1655
1656
for (let { guid, parentGuid, position, needsMerge } of syncableRoots) {
1657
await db.executeCached(
1658
`
1659
INSERT INTO items(guid, parentGuid, kind, needsMerge)
1660
VALUES(:guid, :parentGuid, :kind, :needsMerge)`,
1661
{
1662
guid,
1663
parentGuid,
1664
kind: Ci.mozISyncedBookmarksMerger.KIND_FOLDER,
1665
needsMerge,
1666
}
1667
);
1668
1669
await db.executeCached(
1670
`
1671
INSERT INTO structure(guid, parentGuid, position)
1672
VALUES(:guid, :parentGuid, :position)`,
1673
{ guid, parentGuid, position }
1674
);
1675
}
1676
}
1677
1678
/**
1679
* Creates temporary tables, views, and triggers to apply the mirror to Places.
1680
*
1681
* @param {Sqlite.OpenedConnection} db
1682
* The mirror database connection.
1683
*/
1684
async function initializeTempMirrorEntities(db) {
1685
await db.execute(`CREATE TEMP TABLE changeGuidOps(
1686
localGuid TEXT PRIMARY KEY,
1687
mergedGuid TEXT UNIQUE NOT NULL,
1688
syncStatus INTEGER,
1689
level INTEGER NOT NULL,
1690
lastModifiedMicroseconds INTEGER NOT NULL
1691
) WITHOUT ROWID`);
1692
1693
await db.execute(`
1694
CREATE TEMP TRIGGER changeGuids
1695
AFTER DELETE ON changeGuidOps
1696
BEGIN
1697
/* Record item changed notifications for the updated GUIDs. */
1698
INSERT INTO guidsChanged(itemId, oldGuid, level)
1699
SELECT b.id, OLD.localGuid, OLD.level
1700
FROM moz_bookmarks b
1701
WHERE b.guid = OLD.localGuid;
1702
1703
UPDATE moz_bookmarks SET
1704
guid = OLD.mergedGuid,
1705
lastModified = OLD.lastModifiedMicroseconds,
1706
syncStatus = IFNULL(OLD.syncStatus, syncStatus)
1707
WHERE guid = OLD.localGuid;
1708
END`);
1709
1710
await db.execute(`CREATE TEMP TABLE itemsToApply(
1711
mergedGuid TEXT PRIMARY KEY,
1712
localId INTEGER UNIQUE,
1713
remoteId INTEGER UNIQUE NOT NULL,
1714
remoteGuid TEXT UNIQUE NOT NULL,
1715
newLevel INTEGER NOT NULL,
1716
newType INTEGER NOT NULL,
1717
localDateAddedMicroseconds INTEGER,
1718
remoteDateAddedMicroseconds INTEGER NOT NULL,
1719
lastModifiedMicroseconds INTEGER NOT NULL,
1720
oldTitle TEXT,
1721
newTitle TEXT,
1722
oldPlaceId INTEGER,
1723
newPlaceId INTEGER,
1724
newKeyword TEXT
1725
)`);
1726
1727
await db.execute(`CREATE INDEX existingItems ON itemsToApply(localId)
1728
WHERE localId NOT NULL`);
1729
1730
await db.execute(`CREATE INDEX oldPlaceIds ON itemsToApply(oldPlaceId)
1731
WHERE oldPlaceId NOT NULL`);
1732
1733
await db.execute(`CREATE INDEX newPlaceIds ON itemsToApply(newPlaceId)
1734
WHERE newPlaceId NOT NULL`);
1735
1736
await db.execute(`CREATE INDEX newKeywords ON itemsToApply(newKeyword)
1737
WHERE newKeyword NOT NULL`);
1738
1739
await db.execute(`CREATE TEMP TABLE applyNewLocalStructureOps(
1740
mergedGuid TEXT PRIMARY KEY,
1741
mergedParentGuid TEXT NOT NULL,
1742
position INTEGER NOT NULL,
1743
level INTEGER NOT NULL,
1744
lastModifiedMicroseconds INTEGER NOT NULL
1745
) WITHOUT ROWID`);
1746
1747
await db.execute(`
1748
CREATE TEMP TRIGGER applyNewLocalStructure
1749
AFTER DELETE ON applyNewLocalStructureOps
1750
BEGIN
1751
INSERT INTO itemsMoved(itemId, oldParentId, oldParentGuid, oldPosition,
1752
level)
1753
SELECT b.id, p.id, p.guid, b.position, OLD.level
1754
FROM moz_bookmarks b
1755
JOIN moz_bookmarks p ON p.id = b.parent
1756
WHERE b.guid = OLD.mergedGuid;
1757
1758
UPDATE moz_bookmarks SET
1759
parent = (SELECT id FROM moz_bookmarks
1760
WHERE guid = OLD.mergedParentGuid),
1761
position = OLD.position,
1762
lastModified = OLD.lastModifiedMicroseconds
1763
WHERE guid = OLD.mergedGuid;
1764
END`);
1765
1766
// A view of local bookmark tags. Tags, like keywords, are associated with
1767
// URLs, so two bookmarks with the same URL should have the same tags. Unlike
1768
// keywords, one tag may be associated with many different URLs. Tags are also
1769
// different because they're implemented as bookmarks under the hood. Each tag
1770
// is stored as a folder under the tags root, and tagged URLs are stored as
1771
// untitled bookmarks under these folders. This complexity can be removed once
1772
// bug 424160 lands.
1773
await db.execute(`
1774
CREATE TEMP VIEW localTags(tagEntryId, tagEntryGuid, tagFolderId,
1775
tagFolderGuid, tagEntryPosition, tagEntryType,
1776
tag, placeId, lastModifiedMicroseconds) AS
1777
SELECT b.id, b.guid, p.id, p.guid, b.position, b.type,
1778
p.title, b.fk, b.lastModified
1779
FROM moz_bookmarks b
1780
JOIN moz_bookmarks p ON p.id = b.parent
1781
WHERE b.type = ${PlacesUtils.bookmarks.TYPE_BOOKMARK} AND
1782
p.parent = (SELECT id FROM moz_bookmarks
1783
WHERE guid = '${PlacesUtils.bookmarks.tagsGuid}')`);
1784
1785
// Untags a URL by removing its tag entry.
1786
await db.execute(`
1787
CREATE TEMP TRIGGER untagLocalPlace
1788
INSTEAD OF DELETE ON localTags
1789
BEGIN
1790
/* Record an item removed notification for the tag entry. */
1791
INSERT INTO itemsRemoved(itemId, parentId, position, type, placeId, guid,
1792
parentGuid, isUntagging)
1793
VALUES(OLD.tagEntryId, OLD.tagFolderId, OLD.tagEntryPosition,
1794
OLD.tagEntryType, OLD.placeId, OLD.tagEntryGuid,
1795
OLD.tagFolderGuid, 1);
1796
1797
DELETE FROM moz_bookmarks WHERE id = OLD.tagEntryId;
1798
1799
/* Fix the positions of the sibling tag entries. */
1800
UPDATE moz_bookmarks SET
1801
position = position - 1
1802
WHERE parent = OLD.tagFolderId AND
1803
position > OLD.tagEntryPosition;
1804
END`);
1805
1806
// Tags a URL by creating a tag folder if it doesn't exist, then inserting a
1807
// tag entry for the URL into the tag folder. `NEW.placeId` can be NULL, in
1808
// which case we'll just create the tag folder.
1809
await db.execute(`
1810
CREATE TEMP TRIGGER tagLocalPlace
1811
INSTEAD OF INSERT ON localTags
1812
BEGIN
1813
/* Ensure the tag folder exists. */
1814
INSERT OR IGNORE INTO moz_bookmarks(guid, parent, position, type, title,
1815
dateAdded, lastModified)
1816
VALUES(IFNULL((SELECT b.guid FROM moz_bookmarks b
1817
JOIN moz_bookmarks p ON p.id = b.parent
1818
WHERE b.title = NEW.tag AND
1819
p.guid = '${PlacesUtils.bookmarks.tagsGuid}'),
1820
GENERATE_GUID()),
1821
(SELECT id FROM moz_bookmarks
1822
WHERE guid = '${PlacesUtils.bookmarks.tagsGuid}'),
1823
(SELECT COUNT(*) FROM moz_bookmarks b
1824
JOIN moz_bookmarks p ON p.id = b.parent
1825
WHERE p.guid = '${PlacesUtils.bookmarks.tagsGuid}'),
1826
${PlacesUtils.bookmarks.TYPE_FOLDER}, NEW.tag,
1827
NEW.lastModifiedMicroseconds,
1828
NEW.lastModifiedMicroseconds);
1829
1830
/* Record an item added notification if we created a tag folder.
1831
"CHANGES()" returns the number of rows affected by the INSERT above:
1832
1 if we created the folder, or 0 if the folder already existed. */
1833
INSERT INTO itemsAdded(guid, isTagging)
1834
SELECT b.guid, 1
1835
FROM moz_bookmarks b
1836
JOIN moz_bookmarks p ON p.id = b.parent
1837
WHERE CHANGES() > 0 AND
1838
b.title = NEW.tag AND
1839
p.guid = '${PlacesUtils.bookmarks.tagsGuid}';
1840
1841
/* Add a tag entry for the URL under the tag folder. Omitting the place
1842
ID creates a tag folder without tagging the URL. */
1843
INSERT OR IGNORE INTO moz_bookmarks(guid, parent, position, type, fk,
1844
dateAdded, lastModified)
1845
SELECT IFNULL((SELECT b.guid FROM moz_bookmarks b
1846
JOIN moz_bookmarks p ON p.id = b.parent
1847
WHERE b.fk = NEW.placeId AND
1848
p.title = NEW.tag AND
1849
p.parent = (SELECT id FROM moz_bookmarks
1850
WHERE guid = '${
1851
PlacesUtils.bookmarks.tagsGuid
1852
}')),
1853
GENERATE_GUID()),
1854
(SELECT b.id FROM moz_bookmarks b
1855
JOIN moz_bookmarks p ON p.id = b.parent
1856
WHERE p.guid = '${PlacesUtils.bookmarks.tagsGuid}' AND
1857
b.title = NEW.tag),
1858
(SELECT COUNT(*) FROM moz_bookmarks b
1859
JOIN moz_bookmarks p ON p.id = b.parent
1860
WHERE p.title = NEW.tag AND
1861
p.parent = (SELECT id FROM moz_bookmarks
1862
WHERE guid = '${
1863
PlacesUtils.bookmarks.tagsGuid
1864
}')),
1865
${PlacesUtils.bookmarks.TYPE_BOOKMARK}, NEW.placeId,
1866
NEW.lastModifiedMicroseconds,
1867
NEW.lastModifiedMicroseconds
1868
WHERE NEW.placeId NOT NULL;
1869
1870
/* Record an item added notification for the tag entry. */
1871
INSERT INTO itemsAdded(guid, isTagging)
1872
SELECT b.guid, 1
1873
FROM moz_bookmarks b
1874
JOIN moz_bookmarks p ON p.id = b.parent
1875
WHERE CHANGES() > 0 AND
1876
b.fk = NEW.placeId AND
1877
p.title = NEW.tag AND
1878
p.parent = (SELECT id FROM moz_bookmarks
1879
WHERE guid = '${PlacesUtils.bookmarks.tagsGuid}');
1880
END`);
1881
1882
// Stores properties to pass to `onItem{Added, Changed, Moved, Removed}`
1883
// bookmark observers for new, updated, moved, and deleted items.
1884
await db.execute(`CREATE TEMP TABLE itemsAdded(
1885
guid TEXT PRIMARY KEY,
1886
isTagging BOOLEAN NOT NULL DEFAULT 0,
1887
keywordChanged BOOLEAN NOT NULL DEFAULT 0,
1888
level INTEGER NOT NULL DEFAULT -1
1889
) WITHOUT ROWID`);
1890
1891
await db.execute(`CREATE INDEX addedItemLevels ON itemsAdded(level)`);
1892
1893
await db.execute(`CREATE TEMP TABLE guidsChanged(
1894
itemId INTEGER PRIMARY KEY,
1895
oldGuid TEXT NOT NULL,
1896
level INTEGER NOT NULL DEFAULT -1
1897
)`);
1898
1899
await db.execute(`CREATE INDEX changedGuidLevels ON guidsChanged(level)`);
1900
1901
await db.execute(`CREATE TEMP TABLE itemsChanged(
1902
itemId INTEGER PRIMARY KEY,
1903
oldTitle TEXT,
1904
oldPlaceId INTEGER,
1905
keywordChanged BOOLEAN NOT NULL DEFAULT 0,
1906
level INTEGER NOT NULL DEFAULT -1
1907
)`);
1908
1909
await db.execute(`CREATE INDEX changedItemLevels ON itemsChanged(level)`);
1910
1911
await db.execute(`CREATE TEMP TABLE itemsMoved(
1912
itemId INTEGER PRIMARY KEY,
1913
oldParentId INTEGER NOT NULL,
1914
oldParentGuid TEXT NOT NULL,
1915
oldPosition INTEGER NOT NULL,
1916
level INTEGER NOT NULL DEFAULT -1
1917
)`);
1918
1919
await db.execute(`CREATE INDEX movedItemLevels ON itemsMoved(level)`);
1920
1921
await db.execute(`CREATE TEMP TABLE itemsRemoved(
1922
itemId INTEGER PRIMARY KEY,
1923
guid TEXT NOT NULL,
1924
parentId INTEGER NOT NULL,
1925
position INTEGER NOT NULL,
1926
type INTEGER NOT NULL,
1927
placeId INTEGER,
1928
parentGuid TEXT NOT NULL,
1929
/* We record the original level of the removed item in the tree so that we
1930
can notify children before parents. */
1931
level INTEGER NOT NULL DEFAULT -1,
1932
isUntagging BOOLEAN NOT NULL DEFAULT 0
1933
)`);
1934
1935
await db.execute(
1936
`CREATE INDEX removedItemLevels ON itemsRemoved(level DESC)`
1937
);
1938
1939
// Stores locally changed items staged for upload.
1940
await db.execute(`CREATE TEMP TABLE itemsToUpload(
1941
id INTEGER PRIMARY KEY,
1942
guid TEXT UNIQUE NOT NULL,
1943
syncChangeCounter INTEGER NOT NULL,
1944
isDeleted BOOLEAN NOT NULL DEFAULT 0,
1945
parentGuid TEXT,
1946
parentTitle TEXT,
1947
dateAdded INTEGER, /* In milliseconds. */
1948
type INTEGER,
1949
title TEXT,
1950
placeId INTEGER,
1951
isQuery BOOLEAN NOT NULL DEFAULT 0,
1952
url TEXT,
1953
tagFolderName TEXT,
1954
keyword TEXT,
1955
position INTEGER
1956
)`);
1957
1958
await db.execute(`CREATE TEMP TABLE structureToUpload(
1959
guid TEXT PRIMARY KEY,
1960
parentId INTEGER NOT NULL REFERENCES itemsToUpload(id)
1961
ON DELETE CASCADE,
1962
position INTEGER NOT NULL
1963
) WITHOUT ROWID`);
1964
1965
await db.execute(
1966
`CREATE INDEX parentsToUpload ON structureToUpload(parentId, position)`
1967
);
1968
1969
await db.execute(`CREATE TEMP TABLE tagsToUpload(
1970
id INTEGER REFERENCES itemsToUpload(id)
1971
ON DELETE CASCADE,
1972
tag TEXT,
1973
PRIMARY KEY(id, tag)
1974
) WITHOUT ROWID`);
1975
}
1976
1977
async function resetMirror(db) {
1978
await db.execute(`DELETE FROM meta`);
1979
await db.execute(`DELETE FROM structure`);
1980
await db.execute(`DELETE FROM items`);
1981
await db.execute(`DELETE FROM urls`);
1982
1983
// Since we need to reset the modified times and merge flags for the syncable
1984
// roots, we simply delete and recreate them.
1985
await createMirrorRoots(db);
1986
}
1987
1988
// Converts a Sync record's last modified time to milliseconds.
1989
function determineServerModified(record) {
1990
return Math.max(record.modified * 1000, 0) || 0;
1991
}
1992
1993
// Determines a Sync record's creation date.
1994
function determineDateAdded(record) {
1995
let serverModified = determineServerModified(record);
1996
return PlacesSyncUtils.bookmarks.ratchetTimestampBackwards(
1997
record.dateAdded,
1998
serverModified
1999
);
2000
}
2001
2002
function validateTitle(rawTitle) {
2003
if (typeof rawTitle != "string" || !rawTitle) {
2004
return null;
2005
}
2006
return rawTitle.slice(0, DB_TITLE_LENGTH_MAX);
2007
}
2008
2009
function validateURL(rawURL) {
2010
if (typeof rawURL != "string" || rawURL.length > DB_URL_LENGTH_MAX) {
2011
return null;
2012
}
2013
let url = null;
2014
try {
2015
url = new URL(rawURL);
2016
} catch (ex) {}
2017
return url;
2018
}
2019
2020
function validateKeyword(rawKeyword) {
2021
if (typeof rawKeyword != "string") {
2022
return null;
2023
}
2024
let keyword = rawKeyword.trim();
2025
// Drop empty keywords.
2026
return keyword ? keyword.toLowerCase() : null;
2027
}
2028
2029
function validateTag(rawTag) {
2030
if (typeof rawTag != "string") {
2031
return null;
2032
}
2033
let tag = rawTag.trim();
2034
if (!tag || tag.length > PlacesUtils.bookmarks.MAX_TAG_LENGTH) {
2035
// Drop empty and oversized tags.
2036
return null;
2037
}
2038
return tag;
2039
}
2040
2041
/**
2042
* Measures and logs the time taken to execute a function, using a monotonic
2043
* clock.
2044
*
2045
* @param {String} name
2046
* The name of the operation, used for logging.
2047
* @param {Function} func
2048
* The function to time.
2049
* @param {Function} [recordTiming]
2050
* An optional function with the signature `(time: Number)`, where
2051
* `time` is the measured time.
2052
* @return The return value of the timed function.
2053
*/
2054
async function withTiming(name, func, recordTiming) {
2055
MirrorLog.debug(name);
2056
2057
let startTime = Cu.now();
2058
let result = await func();
2059
let elapsedTime = Cu.now() - startTime;
2060
2061
MirrorLog.debug(`${name} took ${elapsedTime.toFixed(3)}ms`);
2062
if (typeof recordTiming == "function") {
2063
recordTiming(elapsedTime, result);
2064
}
2065
2066
return result;
2067
}
2068
2069
/**
2070
* Fires bookmark and keyword observer notifications for all changes made during
2071
* the merge.
2072
*/
2073
class BookmarkObserverRecorder {
2074
constructor(db, { maxFrecenciesToRecalculate, notifyInStableOrder, signal }) {
2075
this.db = db;
2076
this.maxFrecenciesToRecalculate = maxFrecenciesToRecalculate;
2077
this.notifyInStableOrder = notifyInStableOrder;
2078
this.signal = signal;
2079
this.placesEvents = [];
2080
this.itemRemovedNotifications = [];
2081
this.guidChangedArgs = [];
2082
this.itemMovedArgs = [];
2083
this.itemChangedArgs = [];
2084
this.shouldInvalidateKeywords = false;
2085
}
2086
2087
/**
2088
* Fires observer notifications for all changed items, invalidates the
2089
* livemark cache if necessary, and recalculates frecencies for changed
2090
* URLs. This is called outside the merge transaction.
2091
*/
2092
async notifyAll() {
2093
await this.noteAllChanges();
2094
if (this.shouldInvalidateKeywords) {
2095
await PlacesUtils.keywords.invalidateCachedKeywords();
2096
}
2097
await this.notifyBookmarkObservers();
2098
if (this.signal.aborted) {
2099
throw new SyncedBookmarksMirror.InterruptedError(
2100
"Interrupted before recalculating frecencies for new URLs"
2101
);
2102
}
2103
await updateFrecencies(this.db, this.maxFrecenciesToRecalculate);
2104
}
2105
2106
orderBy(level, parent, position) {
2107
return `ORDER BY ${
2108
this.notifyInStableOrder ? `${level}, ${parent}, ${position}` : level
2109
}`;