Source code

Revision control

Other Tools

1
/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*-
2
* vim: sw=2 ts=2 sts=2 expandtab
3
* This Source Code Form is subject to the terms of the Mozilla Public
4
* License, v. 2.0. If a copy of the MPL was not distributed with this
5
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7
/**
8
* This component handles history and orphans expiration through asynchronous
9
* Storage statements.
10
* Expiration runs:
11
* - At idle, but just once, we stop any other kind of expiration during idle
12
* to preserve batteries in portable devices.
13
* - At shutdown, only if the database is dirty, we should still avoid to
14
* expire too heavily on shutdown.
15
* - On a repeating timer we expire in small chunks.
16
*
17
* Expiration algorithm will adapt itself based on:
18
* - Memory size of the device.
19
* - Status of the database (clean or dirty).
20
*/
21
22
const { XPCOMUtils } = ChromeUtils.import(
24
);
25
const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
26
27
ChromeUtils.defineModuleGetter(
28
this,
29
"PlacesUtils",
31
);
32
33
// Constants
34
35
// Last expiration step should run before the final sync.
36
const TOPIC_PREF_CHANGED = "nsPref:changed";
37
const TOPIC_DEBUG_START_EXPIRATION = "places-debug-start-expiration";
38
const TOPIC_EXPIRATION_FINISHED = "places-expiration-finished";
39
const TOPIC_IDLE_BEGIN = "idle";
40
const TOPIC_IDLE_END = "active";
41
const TOPIC_IDLE_DAILY = "idle-daily";
42
const TOPIC_TESTING_MODE = "testing-mode";
43
const TOPIC_TEST_INTERVAL_CHANGED = "test-interval-changed";
44
45
// Branch for all expiration preferences.
46
const PREF_BRANCH = "places.history.expiration.";
47
48
// Max number of unique URIs to retain in history.
49
// Notice this is a lazy limit. This means we will start to expire if we will
50
// go over it, but we won't ensure that we will stop exactly when we reach it,
51
// instead we will stop after the next expiration step that will bring us
52
// below it.
53
// If this preference does not exist or has a negative value, we will calculate
54
// a limit based on current hardware.
55
const PREF_MAX_URIS = "max_pages";
56
const PREF_MAX_URIS_NOTSET = -1; // Use our internally calculated limit.
57
58
// We save the current unique URIs limit to this pref, to make it available to
59
// other components without having to duplicate the full logic.
60
const PREF_READONLY_CALCULATED_MAX_URIS = "transient_current_max_pages";
61
62
// Seconds between each expiration step.
63
const PREF_INTERVAL_SECONDS = "interval_seconds";
64
const PREF_INTERVAL_SECONDS_NOTSET = 3 * 60;
65
66
// We calculate an optimal database size, based on hardware specs.
67
// This percentage of memory size is used to protect against calculating a too
68
// large database size on systems with small memory.
69
const DATABASE_TO_MEMORY_PERC = 4;
70
// This percentage of disk size is used to protect against calculating a too
71
// large database size on disks with tiny quota or available space.
72
const DATABASE_TO_DISK_PERC = 2;
73
// Maximum size of the optimal database. High-end hardware has plenty of
74
// memory and disk space, but performances don't grow linearly.
75
const DATABASE_MAX_SIZE = 78643200; // 75 MiB
76
// If the physical memory size is bogus, fallback to this.
77
const MEMSIZE_FALLBACK_BYTES = 268435456; // 256 MiB
78
// If the disk available space is bogus, fallback to this.
79
const DISKSIZE_FALLBACK_BYTES = 268435456; // 256 MiB
80
81
// Max number of entries to expire at each expiration step.
82
// This value is globally used for different kind of data we expire, can be
83
// tweaked based on data type. See below in getBoundStatement.
84
const EXPIRE_LIMIT_PER_STEP = 6;
85
// When we run a large expiration step, the above limit is multiplied by this.
86
const EXPIRE_LIMIT_PER_LARGE_STEP_MULTIPLIER = 10;
87
88
// When history is clean or dirty enough we will adapt the expiration algorithm
89
// to be more lazy or more aggressive.
90
// This is done acting on the interval between expiration steps and the number
91
// of expirable items.
92
// 1. Clean history:
93
// We expire at (default interval * EXPIRE_AGGRESSIVITY_MULTIPLIER) the
94
// default number of entries.
95
// 2. Dirty history:
96
// We expire at the default interval, but a greater number of entries
97
// (default number of entries * EXPIRE_AGGRESSIVITY_MULTIPLIER).
98
const EXPIRE_AGGRESSIVITY_MULTIPLIER = 3;
99
100
// This is the average size in bytes of an URI entry in the database.
101
// Magic numbers are determined through analysis of the distribution of a ratio
102
// between number of unique URIs and database size among our users.
103
// Used as a fall back value when it's not possible to calculate the real value.
104
const URIENTRY_AVG_SIZE = 700;
105
106
// Seconds of idle time before starting a larger expiration step.
107
// Notice during idle we stop the expiration timer since we don't want to hurt
108
// stand-by or mobile devices batteries.
109
const IDLE_TIMEOUT_SECONDS = 5 * 60;
110
111
// If the number of pages over history limit is greater than this threshold,
112
// expiration will be more aggressive, to bring back history to a saner size.
113
const OVERLIMIT_PAGES_THRESHOLD = 1000;
114
115
// Milliseconds in a day.
116
const MSECS_PER_DAY = 86400000;
117
118
// When we expire we can use these limits:
119
// - SMALL for usual partial expirations, will expire a small chunk.
120
// - LARGE for idle or shutdown expirations, will expire a large chunk.
121
// - UNLIMITED will expire all the orphans.
122
// - DEBUG will use a known limit, passed along with the debug notification.
123
const LIMIT = {
124
SMALL: 0,
125
LARGE: 1,
126
UNLIMITED: 2,
127
DEBUG: 3,
128
};
129
130
// Represents the status of history database.
131
const STATUS = {
132
CLEAN: 0,
133
DIRTY: 1,
134
UNKNOWN: 2,
135
};
136
137
// Represents actions on which a query will run.
138
const ACTION = {
139
TIMED: 1 << 0, // happens every this._interval
140
TIMED_OVERLIMIT: 1 << 1, // like TIMED but only when history is over limits
141
SHUTDOWN_DIRTY: 1 << 2, // happens at shutdown for DIRTY state
142
IDLE_DIRTY: 1 << 3, // happens on idle for DIRTY state
143
IDLE_DAILY: 1 << 4, // happens once a day on idle
144
DEBUG: 1 << 5, // happens on TOPIC_DEBUG_START_EXPIRATION
145
};
146
147
// The queries we use to expire.
148
const EXPIRATION_QUERIES = {
149
// Some visits can be expired more often than others, cause they are less
150
// useful to the user and can pollute awesomebar results:
151
// 1. urls over 255 chars
152
// 2. redirect sources and downloads
153
// Note: due to the REPLACE option, this should be executed before
154
// QUERY_FIND_VISITS_TO_EXPIRE, that has a more complete result.
155
QUERY_FIND_EXOTIC_VISITS_TO_EXPIRE: {
156
sql: `INSERT INTO expiration_notify (v_id, url, guid, visit_date, reason)
157
SELECT v.id, h.url, h.guid, v.visit_date, "exotic"
158
FROM moz_historyvisits v
159
JOIN moz_places h ON h.id = v.place_id
160
WHERE visit_date < strftime('%s','now','localtime','start of day','-60 days','utc') * 1000000
161
AND ( LENGTH(h.url) > 255 OR v.visit_type = 7 )
162
ORDER BY v.visit_date ASC
163
LIMIT :limit_visits`,
164
actions:
165
ACTION.TIMED_OVERLIMIT |
166
ACTION.IDLE_DIRTY |
167
ACTION.IDLE_DAILY |
168
ACTION.DEBUG,
169
},
170
171
// Finds visits to be expired when history is over the unique pages limit,
172
// otherwise will return nothing.
173
// This explicitly excludes any visits added in the last 7 days, to protect
174
// users with thousands of bookmarks from constantly losing history.
175
QUERY_FIND_VISITS_TO_EXPIRE: {
176
sql: `INSERT INTO expiration_notify
177
(v_id, url, guid, visit_date, expected_results)
178
SELECT v.id, h.url, h.guid, v.visit_date, :limit_visits
179
FROM moz_historyvisits v
180
JOIN moz_places h ON h.id = v.place_id
181
WHERE (SELECT COUNT(*) FROM moz_places) > :max_uris
182
AND visit_date < strftime('%s','now','localtime','start of day','-7 days','utc') * 1000000
183
ORDER BY v.visit_date ASC
184
LIMIT :limit_visits`,
185
actions:
186
ACTION.TIMED_OVERLIMIT |
187
ACTION.IDLE_DIRTY |
188
ACTION.IDLE_DAILY |
189
ACTION.DEBUG,
190
},
191
192
// Removes the previously found visits.
193
QUERY_EXPIRE_VISITS: {
194
sql: `DELETE FROM moz_historyvisits WHERE id IN (
195
SELECT v_id FROM expiration_notify WHERE v_id NOTNULL
196
)`,
197
actions:
198
ACTION.TIMED_OVERLIMIT |
199
ACTION.IDLE_DIRTY |
200
ACTION.IDLE_DAILY |
201
ACTION.DEBUG,
202
},
203
204
// Finds orphan URIs in the database.
205
// Notice we won't notify single removed URIs on History.clear(), so we don't
206
// run this query in such a case, but just delete URIs.
207
// This could run in the middle of adding a visit or bookmark to a new page.
208
// In such a case since it is async, could end up expiring the orphan page
209
// before it actually gets the new visit or bookmark.
210
// Thus, since new pages get frecency -1, we filter on that.
211
QUERY_FIND_URIS_TO_EXPIRE: {
212
sql: `INSERT INTO expiration_notify (p_id, url, guid, visit_date)
213
SELECT h.id, h.url, h.guid, h.last_visit_date
214
FROM moz_places h
215
LEFT JOIN moz_historyvisits v ON h.id = v.place_id
216
WHERE h.last_visit_date IS NULL
217
AND h.foreign_count = 0
218
AND v.id IS NULL
219
AND frecency <> -1
220
LIMIT :limit_uris`,
221
actions:
222
ACTION.TIMED |
223
ACTION.TIMED_OVERLIMIT |
224
ACTION.SHUTDOWN_DIRTY |
225
ACTION.IDLE_DIRTY |
226
ACTION.IDLE_DAILY |
227
ACTION.DEBUG,
228
},
229
230
// Expire found URIs from the database.
231
QUERY_EXPIRE_URIS: {
232
sql: `DELETE FROM moz_places WHERE id IN (
233
SELECT p_id FROM expiration_notify WHERE p_id NOTNULL
234
) AND foreign_count = 0 AND last_visit_date ISNULL`,
235
actions:
236
ACTION.TIMED |
237
ACTION.TIMED_OVERLIMIT |
238
ACTION.SHUTDOWN_DIRTY |
239
ACTION.IDLE_DIRTY |
240
ACTION.IDLE_DAILY |
241
ACTION.DEBUG,
242
},
243
244
// Hosts accumulated during the places delete are updated through a trigger
245
// (see nsPlacesTriggers.h).
246
QUERY_UPDATE_HOSTS: {
247
sql: `DELETE FROM moz_updateoriginsdelete_temp`,
248
actions:
249
ACTION.TIMED |
250
ACTION.TIMED_OVERLIMIT |
251
ACTION.SHUTDOWN_DIRTY |
252
ACTION.IDLE_DIRTY |
253
ACTION.IDLE_DAILY |
254
ACTION.DEBUG,
255
},
256
257
// Expire orphan pages from the icons database.
258
QUERY_EXPIRE_FAVICONS_PAGES: {
259
sql: `DELETE FROM moz_pages_w_icons
260
WHERE page_url_hash NOT IN (
261
SELECT url_hash FROM moz_places
262
)`,
263
actions:
264
ACTION.TIMED_OVERLIMIT |
265
ACTION.SHUTDOWN_DIRTY |
266
ACTION.IDLE_DIRTY |
267
ACTION.IDLE_DAILY |
268
ACTION.DEBUG,
269
},
270
271
// Expire orphan icons from the database.
272
QUERY_EXPIRE_FAVICONS: {
273
sql: `DELETE FROM moz_icons WHERE id IN (
274
SELECT id FROM moz_icons WHERE root = 0
275
EXCEPT
276
SELECT icon_id FROM moz_icons_to_pages
277
)`,
278
actions:
279
ACTION.TIMED_OVERLIMIT |
280
ACTION.SHUTDOWN_DIRTY |
281
ACTION.IDLE_DIRTY |
282
ACTION.IDLE_DAILY |
283
ACTION.DEBUG,
284
},
285
286
// Expire orphan page annotations from the database.
287
QUERY_EXPIRE_ANNOS: {
288
sql: `DELETE FROM moz_annos WHERE id in (
289
SELECT a.id FROM moz_annos a
290
LEFT JOIN moz_places h ON a.place_id = h.id
291
WHERE h.id IS NULL
292
LIMIT :limit_annos
293
)`,
294
actions:
295
ACTION.TIMED |
296
ACTION.TIMED_OVERLIMIT |
297
ACTION.SHUTDOWN_DIRTY |
298
ACTION.IDLE_DIRTY |
299
ACTION.IDLE_DAILY |
300
ACTION.DEBUG,
301
},
302
303
// Expire item annos without a corresponding item id.
304
QUERY_EXPIRE_ITEMS_ANNOS: {
305
sql: `DELETE FROM moz_items_annos WHERE id IN (
306
SELECT a.id FROM moz_items_annos a
307
LEFT JOIN moz_bookmarks b ON a.item_id = b.id
308
WHERE b.id IS NULL
309
LIMIT :limit_annos
310
)`,
311
actions: ACTION.IDLE_DAILY | ACTION.DEBUG,
312
},
313
314
// Expire all annotation names without a corresponding annotation.
315
QUERY_EXPIRE_ANNO_ATTRIBUTES: {
316
sql: `DELETE FROM moz_anno_attributes WHERE id IN (
317
SELECT n.id FROM moz_anno_attributes n
318
LEFT JOIN moz_annos a ON n.id = a.anno_attribute_id
319
LEFT JOIN moz_items_annos t ON n.id = t.anno_attribute_id
320
WHERE a.anno_attribute_id IS NULL
321
AND t.anno_attribute_id IS NULL
322
LIMIT :limit_annos
323
)`,
324
actions:
325
ACTION.SHUTDOWN_DIRTY |
326
ACTION.IDLE_DIRTY |
327
ACTION.IDLE_DAILY |
328
ACTION.DEBUG,
329
},
330
331
// Expire orphan inputhistory.
332
QUERY_EXPIRE_INPUTHISTORY: {
333
sql: `DELETE FROM moz_inputhistory
334
WHERE place_id IN (SELECT p_id FROM expiration_notify)
335
AND place_id IN (
336
SELECT i.place_id FROM moz_inputhistory i
337
LEFT JOIN moz_places h ON h.id = i.place_id
338
WHERE h.id IS NULL
339
LIMIT :limit_inputhistory
340
)`,
341
actions:
342
ACTION.TIMED |
343
ACTION.TIMED_OVERLIMIT |
344
ACTION.SHUTDOWN_DIRTY |
345
ACTION.IDLE_DIRTY |
346
ACTION.IDLE_DAILY |
347
ACTION.DEBUG,
348
},
349
350
// Select entries for notifications.
351
// If p_id is set whole_entry = 1, then we have expired the full page.
352
// Either p_id or v_id are always set.
353
QUERY_SELECT_NOTIFICATIONS: {
354
sql: `SELECT url, guid, MAX(visit_date) AS visit_date,
355
MAX(IFNULL(MIN(p_id, 1), MIN(v_id, 0))) AS whole_entry,
356
MAX(expected_results) AS expected_results,
357
(SELECT MAX(visit_date) FROM expiration_notify
358
WHERE reason = "expired" AND url = n.url AND p_id ISNULL
359
) AS most_recent_expired_visit
360
FROM expiration_notify n
361
GROUP BY url`,
362
actions:
363
ACTION.TIMED |
364
ACTION.TIMED_OVERLIMIT |
365
ACTION.SHUTDOWN_DIRTY |
366
ACTION.IDLE_DIRTY |
367
ACTION.IDLE_DAILY |
368
ACTION.DEBUG,
369
},
370
371
// Empty the notifications table.
372
QUERY_DELETE_NOTIFICATIONS: {
373
sql: "DELETE FROM expiration_notify",
374
actions:
375
ACTION.TIMED |
376
ACTION.TIMED_OVERLIMIT |
377
ACTION.SHUTDOWN_DIRTY |
378
ACTION.IDLE_DIRTY |
379
ACTION.IDLE_DAILY |
380
ACTION.DEBUG,
381
},
382
};
383
384
/**
385
* Sends a bookmarks notification through the given observers.
386
*
387
* @param observers
388
* array of nsINavBookmarkObserver objects.
389
* @param notification
390
* the notification name.
391
* @param args
392
* array of arguments to pass to the notification.
393
*/
394
function notify(observers, notification, args = []) {
395
for (let observer of observers) {
396
try {
397
observer[notification](...args);
398
} catch (ex) {}
399
}
400
}
401
402
// nsPlacesExpiration definition
403
404
function nsPlacesExpiration() {
405
// Smart Getters
406
407
XPCOMUtils.defineLazyGetter(this, "_db", function() {
408
let db = PlacesUtils.history.DBConnection;
409
410
// Create the temporary notifications table.
411
let stmt = db.createAsyncStatement(
412
`CREATE TEMP TABLE expiration_notify (
413
id INTEGER PRIMARY KEY
414
, v_id INTEGER
415
, p_id INTEGER
416
, url TEXT NOT NULL
417
, guid TEXT NOT NULL
418
, visit_date INTEGER
419
, expected_results INTEGER NOT NULL DEFAULT 0
420
, reason TEXT NOT NULL DEFAULT "expired"
421
)`
422
);
423
stmt.executeAsync();
424
stmt.finalize();
425
426
return db;
427
});
428
429
XPCOMUtils.defineLazyServiceGetter(
430
this,
431
"_idle",
432
"@mozilla.org/widget/idleservice;1",
433
"nsIIdleService"
434
);
435
436
this._prefBranch = Services.prefs.getBranch(PREF_BRANCH);
437
438
this._loadPrefsPromise = this._loadPrefs().then(() => {
439
// Observe our preferences branch for changes.
440
this._prefBranch.addObserver("", this, true);
441
442
// Create our expiration timer.
443
this._newTimer();
444
}, Cu.reportError);
445
446
// Register topic observers.
447
Services.obs.addObserver(this, TOPIC_DEBUG_START_EXPIRATION, true);
448
Services.obs.addObserver(this, TOPIC_IDLE_DAILY, true);
449
450
// Block shutdown.
451
let shutdownClient = PlacesUtils.history.connectionShutdownClient.jsclient;
452
shutdownClient.addBlocker("Places Expiration: shutdown", () => {
453
if (this._shuttingDown) {
454
return;
455
}
456
this._shuttingDown = true;
457
this.expireOnIdle = false;
458
if (this._timer) {
459
this._timer.cancel();
460
this._timer = null;
461
}
462
// If the database is dirty, we want to expire some entries, to speed up
463
// the expiration process.
464
if (this.status == STATUS.DIRTY) {
465
this._expireWithActionAndLimit(ACTION.SHUTDOWN_DIRTY, LIMIT.LARGE);
466
}
467
this._finalizeInternalStatements();
468
});
469
}
470
471
nsPlacesExpiration.prototype = {
472
observe: function PEX_observe(aSubject, aTopic, aData) {
473
if (this._shuttingDown) {
474
return;
475
}
476
477
if (aTopic == TOPIC_PREF_CHANGED) {
478
this._loadPrefsPromise = this._loadPrefs().then(() => {
479
if (aData == PREF_INTERVAL_SECONDS) {
480
// Renew the timer with the new interval value.
481
this._newTimer();
482
}
483
}, Cu.reportError);
484
} else if (aTopic == TOPIC_DEBUG_START_EXPIRATION) {
485
// The passed-in limit is the maximum number of visits to expire when
486
// history is over capacity. Mind to correctly handle the NaN value.
487
let limit = parseInt(aData);
488
if (limit == -1) {
489
// Everything should be expired without any limit. If history is over
490
// capacity then all existing visits will be expired.
491
// Should only be used in tests, since may cause dataloss.
492
this._expireWithActionAndLimit(ACTION.DEBUG, LIMIT.UNLIMITED);
493
} else if (limit > 0) {
494
// The number of expired visits is limited by this amount. It may be
495
// used for testing purposes, like checking that limited queries work.
496
this._debugLimit = limit;
497
this._expireWithActionAndLimit(ACTION.DEBUG, LIMIT.DEBUG);
498
} else {
499
// Any other value is intended as a 0 limit, that means no visits
500
// will be expired. Even if this doesn't touch visits, it will remove
501
// any orphan pages, icons, annotations and similar from the database,
502
// so it may be used for cleanup purposes.
503
this._debugLimit = -1;
504
this._expireWithActionAndLimit(ACTION.DEBUG, LIMIT.DEBUG);
505
}
506
} else if (aTopic == TOPIC_IDLE_BEGIN) {
507
// Stop the expiration timer. We don't want to keep up expiring on idle
508
// to preserve batteries on mobile devices and avoid killing stand-by.
509
if (this._timer) {
510
this._timer.cancel();
511
this._timer = null;
512
}
513
if (this.expireOnIdle) {
514
this._expireWithActionAndLimit(ACTION.IDLE_DIRTY, LIMIT.LARGE);
515
}
516
} else if (aTopic == TOPIC_IDLE_END) {
517
// Restart the expiration timer.
518
if (!this._timer) {
519
this._newTimer();
520
}
521
} else if (aTopic == TOPIC_IDLE_DAILY) {
522
this._expireWithActionAndLimit(ACTION.IDLE_DAILY, LIMIT.LARGE);
523
} else if (aTopic == TOPIC_TESTING_MODE) {
524
this._testingMode = true;
525
} else if (aTopic == PlacesUtils.TOPIC_INIT_COMPLETE) {
526
// Ideally we'd add this observer only when notifications start being
527
// triggered. However, that's difficult to work out, so we do it on
528
// TOPIC_INIT_COMPLETE which means we have to take the hit of initializing
529
// this service slightly earlier.
530
PlacesUtils.history.addObserver(this, true);
531
}
532
},
533
534
// nsINavHistoryObserver
535
536
_inBatchMode: false,
537
onBeginUpdateBatch: function PEX_onBeginUpdateBatch() {
538
this._inBatchMode = true;
539
540
// We do not want to expire while we are doing batch work.
541
if (this._timer) {
542
this._timer.cancel();
543
this._timer = null;
544
}
545
},
546
547
onEndUpdateBatch: function PEX_onEndUpdateBatch() {
548
this._inBatchMode = false;
549
550
// Restore timer.
551
if (!this._timer) {
552
this._newTimer();
553
}
554
},
555
556
onClearHistory: function PEX_onClearHistory() {
557
// History status is clean after a clear history.
558
this.status = STATUS.CLEAN;
559
},
560
561
onTitleChanged() {},
562
onDeleteURI() {},
563
onPageChanged() {},
564
onDeleteVisits() {},
565
566
// nsITimerCallback
567
568
notify: function PEX_timerCallback() {
569
// Check if we are over history capacity, if so visits must be expired.
570
this._getPagesStats(aPagesCount => {
571
let overLimitPages = aPagesCount - this._urisLimit;
572
this._overLimit = overLimitPages > 0;
573
let action = this._overLimit ? ACTION.TIMED_OVERLIMIT : ACTION.TIMED;
574
// Adapt expiration aggressivity to the number of pages over the limit.
575
let limit =
576
overLimitPages > OVERLIMIT_PAGES_THRESHOLD ? LIMIT.LARGE : LIMIT.SMALL;
577
// Run at the first idle, or after a minute, whatever comes first.
578
Services.tm.idleDispatchToMainThread(() => {
579
this._expireWithActionAndLimit(action, limit);
580
}, 60000);
581
});
582
},
583
584
// mozIStorageStatementCallback
585
586
handleResult: function PEX_handleResult(aResultSet) {
587
// We don't want to notify after shutdown.
588
if (this._shuttingDown) {
589
return;
590
}
591
592
let row;
593
while ((row = aResultSet.getNextRow())) {
594
// expected_results is set to the number of expected visits by
595
// QUERY_FIND_VISITS_TO_EXPIRE. We decrease that counter for each found
596
// visit and if it reaches zero we mark the database as dirty, since all
597
// the expected visits were expired, so it's likely the next run will
598
// find more.
599
let expectedResults = row.getResultByName("expected_results");
600
if (expectedResults > 0) {
601
if (!("_expectedResultsCount" in this)) {
602
this._expectedResultsCount = expectedResults;
603
}
604
if (this._expectedResultsCount > 0) {
605
this._expectedResultsCount--;
606
}
607
}
608
609
let uri = Services.io.newURI(row.getResultByName("url"));
610
let guid = row.getResultByName("guid");
611
let visitDate = row.getResultByName("visit_date");
612
let wholeEntry = row.getResultByName("whole_entry");
613
let mostRecentExpiredVisit = row.getResultByName(
614
"most_recent_expired_visit"
615
);
616
let reason = Ci.nsINavHistoryObserver.REASON_EXPIRED;
617
let observers = PlacesUtils.history.getObservers();
618
619
if (mostRecentExpiredVisit) {
620
let days = parseInt(
621
(Date.now() - mostRecentExpiredVisit / 1000) / MSECS_PER_DAY
622
);
623
if (!this._mostRecentExpiredVisitDays) {
624
this._mostRecentExpiredVisitDays = days;
625
} else if (days < this._mostRecentExpiredVisitDays) {
626
this._mostRecentExpiredVisitDays = days;
627
}
628
}
629
630
// Dispatch expiration notifications to history.
631
if (wholeEntry) {
632
notify(observers, "onDeleteURI", [uri, guid, reason]);
633
} else {
634
notify(observers, "onDeleteVisits", [
635
uri,
636
visitDate > 0,
637
guid,
638
reason,
639
0,
640
]);
641
}
642
}
643
},
644
645
handleError: function PEX_handleError(aError) {
646
Cu.reportError(
647
"Async statement execution returned with '" +
648
aError.result +
649
"', '" +
650
aError.message +
651
"'"
652
);
653
},
654
655
// Number of expiration steps needed to reach a CLEAN status.
656
_telemetrySteps: 1,
657
handleCompletion: function PEX_handleCompletion(aReason) {
658
if (aReason == Ci.mozIStorageStatementCallback.REASON_FINISHED) {
659
if (this._mostRecentExpiredVisitDays) {
660
try {
661
Services.telemetry
662
.getHistogramById("PLACES_MOST_RECENT_EXPIRED_VISIT_DAYS")
663
.add(this._mostRecentExpiredVisitDays);
664
} catch (ex) {
665
Cu.reportError("Unable to report telemetry.");
666
} finally {
667
delete this._mostRecentExpiredVisitDays;
668
}
669
}
670
671
if ("_expectedResultsCount" in this) {
672
// Adapt the aggressivity of steps based on the status of history.
673
// A dirty history will return all the entries we are expecting bringing
674
// our countdown to zero, while a clean one will not.
675
let oldStatus = this.status;
676
this.status =
677
this._expectedResultsCount == 0 ? STATUS.DIRTY : STATUS.CLEAN;
678
679
// Collect or send telemetry data.
680
if (this.status == STATUS.DIRTY) {
681
this._telemetrySteps++;
682
} else {
683
// Avoid reporting the common cases where the database is clean, or
684
// a single step is needed.
685
if (oldStatus == STATUS.DIRTY) {
686
try {
687
Services.telemetry
688
.getHistogramById("PLACES_EXPIRATION_STEPS_TO_CLEAN2")
689
.add(this._telemetrySteps);
690
} catch (ex) {
691
Cu.reportError("Unable to report telemetry.");
692
}
693
}
694
this._telemetrySteps = 1;
695
}
696
697
delete this._expectedResultsCount;
698
}
699
700
// Dispatch a notification that expiration has finished.
701
Services.obs.notifyObservers(null, TOPIC_EXPIRATION_FINISHED);
702
}
703
},
704
705
// nsPlacesExpiration
706
707
_urisLimit: PREF_MAX_URIS_NOTSET,
708
_interval: PREF_INTERVAL_SECONDS_NOTSET,
709
_shuttingDown: false,
710
711
_status: STATUS.UNKNOWN,
712
set status(aNewStatus) {
713
if (aNewStatus != this._status) {
714
// If status changes we should restart the timer.
715
this._status = aNewStatus;
716
this._newTimer();
717
// If needed add/remove the cleanup step on idle. We want to expire on
718
// idle only if history is dirty, to preserve mobile devices batteries.
719
this.expireOnIdle = aNewStatus == STATUS.DIRTY;
720
}
721
return aNewStatus;
722
},
723
get status() {
724
return this._status;
725
},
726
727
_isIdleObserver: false,
728
_expireOnIdle: false,
729
set expireOnIdle(aExpireOnIdle) {
730
// Observe idle regardless aExpireOnIdle, since we always want to stop
731
// timed expiration on idle, to preserve mobile battery life.
732
if (!this._isIdleObserver && !this._shuttingDown) {
733
this._idle.addIdleObserver(this, IDLE_TIMEOUT_SECONDS);
734
this._isIdleObserver = true;
735
} else if (this._isIdleObserver && this._shuttingDown) {
736
this._idle.removeIdleObserver(this, IDLE_TIMEOUT_SECONDS);
737
this._isIdleObserver = false;
738
}
739
740
// If running a debug expiration we need full control of what happens
741
// but idle cleanup could activate in the middle, since tinderboxes are
742
// permanently idle. That would cause unexpected oranges, so disable it.
743
if (this._debugLimit !== undefined) {
744
this._expireOnIdle = false;
745
} else {
746
this._expireOnIdle = aExpireOnIdle;
747
}
748
return this._expireOnIdle;
749
},
750
get expireOnIdle() {
751
return this._expireOnIdle;
752
},
753
754
async _loadPrefs() {
755
// Get the user's limit, if it was set.
756
this._urisLimit = this._prefBranch.getIntPref(
757
PREF_MAX_URIS,
758
PREF_MAX_URIS_NOTSET
759
);
760
if (this._urisLimit < 0) {
761
// Some testing code expects a pref change to be synchronous, so
762
// temporarily set this to a large value, while we asynchronously update
763
// to the correct value.
764
this._urisLimit = 100000;
765
766
// The user didn't specify a custom limit, so we calculate the number of
767
// unique places that may fit an optimal database size on this hardware.
768
// Oldest pages over this threshold will be expired.
769
let memSizeBytes = MEMSIZE_FALLBACK_BYTES;
770
try {
771
// Limit the size on systems with small memory.
772
memSizeBytes = Services.sysinfo.getProperty("memsize");
773
} catch (ex) {}
774
if (memSizeBytes <= 0) {
775
memSizeBytes = MEMSIZE_FALLBACK_BYTES;
776
}
777
778
let diskAvailableBytes = DISKSIZE_FALLBACK_BYTES;
779
try {
780
// Protect against a full disk or tiny quota.
781
let dbFile = this._db.databaseFile;
782
dbFile.QueryInterface(Ci.nsIFile);
783
diskAvailableBytes = dbFile.diskSpaceAvailable;
784
} catch (ex) {}
785
if (diskAvailableBytes <= 0) {
786
diskAvailableBytes = DISKSIZE_FALLBACK_BYTES;
787
}
788
789
let optimalDatabaseSize = Math.min(
790
(memSizeBytes * DATABASE_TO_MEMORY_PERC) / 100,
791
(diskAvailableBytes * DATABASE_TO_DISK_PERC) / 100,
792
DATABASE_MAX_SIZE
793
);
794
795
// Calculate avg size of a URI in the database.
796
let db;
797
try {
798
db = await PlacesUtils.promiseDBConnection();
799
if (db) {
800
let row = (await db.execute(`SELECT * FROM pragma_page_size(),
801
pragma_page_count(),
802
pragma_freelist_count(),
803
(SELECT count(*) FROM moz_places)`))[0];
804
let pageSize = row.getResultByIndex(0);
805
let pageCount = row.getResultByIndex(1);
806
let freelistCount = row.getResultByIndex(2);
807
let uriCount = row.getResultByIndex(3);
808
let dbSize = (pageCount - freelistCount) * pageSize;
809
let avgURISize = Math.ceil(dbSize / uriCount);
810
// For new profiles this value may be too large, due to the Sqlite header,
811
// or Infinity when there are no pages. Thus we must limit it.
812
if (avgURISize > URIENTRY_AVG_SIZE * 3) {
813
avgURISize = URIENTRY_AVG_SIZE;
814
}
815
this._urisLimit = Math.ceil(optimalDatabaseSize / avgURISize);
816
}
817
} catch (ex) {
818
// We may have been initialized late in the shutdown process, maybe
819
// by a call to clear history on shutdown.
820
// If we're unable to get a connection clone, we'll just proceed with
821
// the default value, it should not be critical at this point in the
822
// application life-cycle.
823
}
824
}
825
826
// Expose the calculated limit to other components.
827
this._prefBranch.setIntPref(
828
PREF_READONLY_CALCULATED_MAX_URIS,
829
this._urisLimit
830
);
831
832
// Get the expiration interval value.
833
this._interval = this._prefBranch.getIntPref(
834
PREF_INTERVAL_SECONDS,
835
PREF_INTERVAL_SECONDS_NOTSET
836
);
837
if (this._interval <= 0) {
838
this._interval = PREF_INTERVAL_SECONDS_NOTSET;
839
}
840
},
841
842
/**
843
* Evaluates the real number of pages in the database and the value currently
844
* used by the SQLite query planner.
845
*
846
* @param aCallback
847
* invoked on success, function (aPagesCount).
848
*/
849
_getPagesStats: function PEX__getPagesStats(aCallback) {
850
if (!this._cachedStatements.LIMIT_COUNT) {
851
this._cachedStatements.LIMIT_COUNT = this._db.createAsyncStatement(
852
`SELECT COUNT(*) FROM moz_places`
853
);
854
}
855
this._cachedStatements.LIMIT_COUNT.executeAsync({
856
_pagesCount: 0,
857
handleResult(aResults) {
858
let row = aResults.getNextRow();
859
this._pagesCount = row.getResultByIndex(0);
860
},
861
handleCompletion(aReason) {
862
if (aReason == Ci.mozIStorageStatementCallback.REASON_FINISHED) {
863
aCallback(this._pagesCount);
864
}
865
},
866
handleError(aError) {
867
Cu.reportError(
868
"Async statement execution returned with '" +
869
aError.result +
870
"', '" +
871
aError.message +
872
"'"
873
);
874
},
875
});
876
},
877
878
/**
879
* Execute async statements to expire with the specified queries.
880
*
881
* @param aAction
882
* The ACTION we are expiring for. See the ACTION const for values.
883
* @param aLimit
884
* Whether to use small, large or no limits when expiring. See the
885
* LIMIT const for values.
886
*/
887
_expireWithActionAndLimit: function PEX__expireWithActionAndLimit(
888
aAction,
889
aLimit
890
) {
891
(async () => {
892
// Ensure that we'll run statements with the most up-to-date pref values.
893
// On shutdown we cannot do this, since we must enqueue the expiration
894
// statements synchronously before the connection goes away.
895
// TODO (Bug 1275878): handle this properly through a shutdown blocker.
896
if (!this._shuttingDown) {
897
await this._loadPrefsPromise;
898
}
899
900
// Skip expiration during batch mode.
901
if (this._inBatchMode) {
902
return;
903
}
904
// Don't try to further expire after shutdown.
905
if (this._shuttingDown && aAction != ACTION.SHUTDOWN_DIRTY) {
906
return;
907
}
908
909
let boundStatements = [];
910
for (let queryType in EXPIRATION_QUERIES) {
911
if (EXPIRATION_QUERIES[queryType].actions & aAction) {
912
boundStatements.push(
913
this._getBoundStatement(queryType, aLimit, aAction)
914
);
915
}
916
}
917
918
// Execute statements asynchronously in a transaction.
919
this._db.executeAsync(boundStatements, this);
920
})().catch(Cu.reportError);
921
},
922
923
/**
924
* Finalizes all of our mozIStorageStatements so we can properly close the
925
* database.
926
*/
927
_finalizeInternalStatements: function PEX__finalizeInternalStatements() {
928
for (let queryType in this._cachedStatements) {
929
let stmt = this._cachedStatements[queryType];
930
stmt.finalize();
931
}
932
},
933
934
/**
935
* Generate the statement used for expiration.
936
*
937
* @param aQueryType
938
* Type of the query to build statement for.
939
* @param aLimit
940
* Whether to use small, large or no limits when expiring. See the
941
* LIMIT const for values.
942
* @param aAction
943
* Current action causing the expiration. See the ACTION const.
944
*/
945
_cachedStatements: {},
946
_getBoundStatement: function PEX__getBoundStatement(
947
aQueryType,
948
aLimit,
949
aAction
950
) {
951
// Statements creation can be expensive, so we want to cache them.
952
let stmt = this._cachedStatements[aQueryType];
953
if (stmt === undefined) {
954
stmt = this._cachedStatements[aQueryType] = this._db.createAsyncStatement(
955
EXPIRATION_QUERIES[aQueryType].sql
956
);
957
}
958
959
let baseLimit;
960
switch (aLimit) {
961
case LIMIT.UNLIMITED:
962
baseLimit = -1;
963
break;
964
case LIMIT.SMALL:
965
baseLimit = EXPIRE_LIMIT_PER_STEP;
966
break;
967
case LIMIT.LARGE:
968
baseLimit =
969
EXPIRE_LIMIT_PER_STEP * EXPIRE_LIMIT_PER_LARGE_STEP_MULTIPLIER;
970
break;
971
case LIMIT.DEBUG:
972
baseLimit = this._debugLimit;
973
break;
974
}
975
if (
976
this.status == STATUS.DIRTY &&
977
aAction != ACTION.DEBUG &&
978
baseLimit > 0
979
) {
980
baseLimit *= EXPIRE_AGGRESSIVITY_MULTIPLIER;
981
}
982
983
// Bind the appropriate parameters.
984
let params = stmt.params;
985
switch (aQueryType) {
986
case "QUERY_FIND_EXOTIC_VISITS_TO_EXPIRE":
987
// Avoid expiring all visits in case of an unlimited debug expiration,
988
// just remove orphans instead.
989
params.limit_visits =
990
aLimit == LIMIT.DEBUG && baseLimit == -1 ? 0 : baseLimit;
991
break;
992
case "QUERY_FIND_VISITS_TO_EXPIRE":
993
params.max_uris = this._urisLimit;
994
// Avoid expiring all visits in case of an unlimited debug expiration,
995
// just remove orphans instead.
996
params.limit_visits =
997
aLimit == LIMIT.DEBUG && baseLimit == -1 ? 0 : baseLimit;
998
break;
999
case "QUERY_FIND_URIS_TO_EXPIRE":
1000
params.limit_uris = baseLimit;
1001
break;
1002
case "QUERY_EXPIRE_ANNOS":
1003
// Each page may have multiple annos.
1004
params.limit_annos = baseLimit * EXPIRE_AGGRESSIVITY_MULTIPLIER;
1005
break;
1006
case "QUERY_EXPIRE_ITEMS_ANNOS":
1007
params.limit_annos = baseLimit;
1008
break;
1009
case "QUERY_EXPIRE_ANNO_ATTRIBUTES":
1010
params.limit_annos = baseLimit;
1011
break;
1012
case "QUERY_EXPIRE_INPUTHISTORY":
1013
params.limit_inputhistory = baseLimit;
1014
break;
1015
}
1016
1017
return stmt;
1018
},
1019
1020
/**
1021
* Creates a new timer based on this._interval.
1022
*
1023
* @return a REPEATING_SLACK nsITimer that runs every this._interval.
1024
*/
1025
_newTimer: function PEX__newTimer() {
1026
if (this._timer) {
1027
this._timer.cancel();
1028
}
1029
if (this._shuttingDown) {
1030
return undefined;
1031
}
1032
let interval =
1033
this.status != STATUS.DIRTY
1034
? this._interval * EXPIRE_AGGRESSIVITY_MULTIPLIER
1035
: this._interval;
1036
1037
let timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
1038
timer.initWithCallback(
1039
this,
1040
interval * 1000,
1041
Ci.nsITimer.TYPE_REPEATING_SLACK_LOW_PRIORITY
1042
);
1043
if (this._testingMode) {
1044
Services.obs.notifyObservers(null, TOPIC_TEST_INTERVAL_CHANGED, interval);
1045
}
1046
return (this._timer = timer);
1047
},
1048
1049
// nsISupports
1050
1051
classID: Components.ID("705a423f-2f69-42f3-b9fe-1517e0dee56f"),
1052
1053
QueryInterface: ChromeUtils.generateQI([
1054
Ci.nsIObserver,
1055
Ci.nsINavHistoryObserver,
1056
Ci.nsITimerCallback,
1057
Ci.mozIStorageStatementCallback,
1058
Ci.nsISupportsWeakReference,
1059
]),
1060
};
1061
1062
// Module Registration
1063
1064
var EXPORTED_SYMBOLS = ["nsPlacesExpiration"];