Source code

Revision control

Other Tools

1
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2
* vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ :
3
* This Source Code Form is subject to the terms of the Mozilla Public
4
* License, v. 2.0. If a copy of the MPL was not distributed with this
5
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7
/**
8
* A watchdog designed to terminate shutdown if it lasts too long.
9
*
10
* This watchdog is designed as a worst-case problem container for the
11
* common case in which Firefox just won't shutdown.
12
*
13
* We spawn a thread during quit-application. If any of the shutdown
14
* steps takes more than n milliseconds (63000 by default), kill the
15
* process as fast as possible, without any cleanup.
16
*/
17
18
#include "nsTerminator.h"
19
20
#include "prthread.h"
21
#include "prmon.h"
22
#include "plstr.h"
23
#include "prio.h"
24
25
#include "nsString.h"
26
#include "nsServiceManagerUtils.h"
27
#include "nsDirectoryServiceUtils.h"
28
#include "nsAppDirectoryServiceDefs.h"
29
30
#include "nsIObserverService.h"
31
#include "nsExceptionHandler.h"
32
#include "GeckoProfiler.h"
33
#include "nsThreadUtils.h"
34
#include "nsXULAppAPI.h"
35
36
#if defined(XP_WIN)
37
# include <windows.h>
38
#else
39
# include <unistd.h>
40
#endif
41
42
#include "mozilla/ArrayUtils.h"
43
#include "mozilla/Atomics.h"
44
#include "mozilla/Attributes.h"
45
#include "mozilla/DebugOnly.h"
46
#include "mozilla/IntentionalCrash.h"
47
#include "mozilla/MemoryChecking.h"
48
#include "mozilla/Preferences.h"
49
#include "mozilla/Services.h"
50
#include "mozilla/UniquePtr.h"
51
#include "mozilla/Unused.h"
52
#include "mozilla/Telemetry.h"
53
54
#include "mozilla/dom/workerinternals/RuntimeService.h"
55
56
// Normally, the number of milliseconds that AsyncShutdown waits until
57
// it decides to crash is specified as a preference. We use the
58
// following value as a fallback if for some reason the preference is
59
// absent.
60
#define FALLBACK_ASYNCSHUTDOWN_CRASH_AFTER_MS 60000
61
62
// Additional number of milliseconds to wait until we decide to exit
63
// forcefully.
64
#define ADDITIONAL_WAIT_BEFORE_CRASH_MS 3000
65
66
namespace mozilla {
67
68
namespace {
69
70
/**
71
* A step during shutdown.
72
*
73
* Shutdown is divided in steps, which all map to an observer
74
* notification. The duration of a step is defined as the number of
75
* ticks between the time we receive a notification and the next one.
76
*/
77
struct ShutdownStep {
78
char const* const mTopic;
79
int mTicks;
80
81
constexpr explicit ShutdownStep(const char* const topic)
82
: mTopic(topic), mTicks(-1) {}
83
};
84
85
static ShutdownStep sShutdownSteps[] = {
86
ShutdownStep("quit-application"),
87
ShutdownStep("profile-change-teardown"),
88
ShutdownStep("profile-before-change"),
89
ShutdownStep("xpcom-will-shutdown"),
90
ShutdownStep("xpcom-shutdown"),
91
};
92
93
Atomic<bool> sShutdownNotified;
94
95
// Utility function: create a thread that is non-joinable,
96
// does not prevent the process from terminating, is never
97
// cooperatively scheduled, and uses a default stack size.
98
PRThread* CreateSystemThread(void (*start)(void* arg), void* arg) {
99
PRThread* thread =
100
PR_CreateThread(PR_SYSTEM_THREAD, /* This thread will not prevent the
101
process from terminating */
102
start, arg, PR_PRIORITY_LOW,
103
PR_GLOBAL_THREAD, /* Make sure that the thread is never
104
cooperatively scheduled */
105
PR_UNJOINABLE_THREAD, 0 /* Use default stack size */
106
);
107
MOZ_LSAN_INTENTIONALLY_LEAK_OBJECT(
108
thread); // This pointer will never be deallocated.
109
return thread;
110
}
111
112
////////////////////////////////////////////
113
//
114
// The watchdog
115
//
116
// This nspr thread is in charge of crashing the process if any stage of
117
// shutdown lasts more than some predefined duration. As a side-effect, it
118
// measures the duration of each stage of shutdown.
119
//
120
121
// The heartbeat of the operation.
122
//
123
// Main thread:
124
//
125
// * Whenever a shutdown step has been completed, the main thread
126
// swaps gHeartbeat to 0 to mark that the shutdown process is still
127
// progressing. The value swapped away indicates the number of ticks
128
// it took for the shutdown step to advance.
129
//
130
// Watchdog thread:
131
//
132
// * Every tick, the watchdog thread increments gHearbeat atomically.
133
//
134
// A note about precision:
135
// Since gHeartbeat is generally reset to 0 between two ticks, this means
136
// that gHeartbeat stays at 0 less than one tick. Consequently, values
137
// extracted from gHeartbeat must be considered rounded up.
138
Atomic<uint32_t> gHeartbeat(0);
139
140
struct Options {
141
/**
142
* How many ticks before we should crash the process.
143
*/
144
uint32_t crashAfterTicks;
145
};
146
147
/**
148
* Entry point for the watchdog thread
149
*/
150
void RunWatchdog(void* arg) {
151
NS_SetCurrentThreadName("Shutdown Hang Terminator");
152
153
// Let's copy and deallocate options, that's one less leak to worry
154
// about.
155
UniquePtr<Options> options((Options*)arg);
156
uint32_t crashAfterTicks = options->crashAfterTicks;
157
options = nullptr;
158
159
const uint32_t timeToLive = crashAfterTicks;
160
while (true) {
161
//
162
// We do not want to sleep for the entire duration,
163
// as putting the computer to sleep would suddenly
164
// cause us to timeout on wakeup.
165
//
166
// Rather, we prefer sleeping for at most 1 second
167
// at a time. If the computer sleeps then wakes up,
168
// we have lost at most one second, which is much
169
// more reasonable.
170
//
171
#if defined(XP_WIN)
172
Sleep(1000 /* ms */);
173
#else
174
usleep(1000000 /* usec */);
175
#endif
176
177
if (gHeartbeat++ < timeToLive) {
178
continue;
179
}
180
181
NoteIntentionalCrash(XRE_GetProcessTypeString());
182
183
// The shutdown steps are not completed yet. Let's report the last one.
184
if (!sShutdownNotified) {
185
const char* lastStep = nullptr;
186
for (size_t i = 0; i < ArrayLength(sShutdownSteps); ++i) {
187
if (sShutdownSteps[i].mTicks == -1) {
188
break;
189
}
190
lastStep = sShutdownSteps[i].mTopic;
191
}
192
193
if (lastStep) {
194
nsCString msg;
195
msg.AppendPrintf(
196
"Shutdown hanging at step %s. "
197
"Something is blocking the main-thread.",
198
lastStep);
199
// This string will be leaked.
200
MOZ_CRASH_UNSAFE(strdup(msg.BeginReading()));
201
}
202
203
MOZ_CRASH("Shutdown hanging before starting.");
204
}
205
206
// Maybe some workers are blocking the shutdown.
207
mozilla::dom::workerinternals::RuntimeService* runtimeService =
208
mozilla::dom::workerinternals::RuntimeService::GetService();
209
if (runtimeService) {
210
runtimeService->CrashIfHanging();
211
}
212
213
// Shutdown is apparently dead. Crash the process.
214
CrashReporter::SetMinidumpAnalysisAllThreads();
215
216
MOZ_CRASH("Shutdown too long, probably frozen, causing a crash.");
217
}
218
}
219
220
////////////////////////////////////////////
221
//
222
// Writer thread
223
//
224
// This nspr thread is in charge of writing to disk statistics produced by the
225
// watchdog thread and collected by the main thread. Note that we use a nspr
226
// thread rather than usual XPCOM I/O simply because we outlive XPCOM and its
227
// threads.
228
//
229
230
// Utility class, used by UniquePtr<> to close nspr files.
231
class PR_CloseDelete {
232
public:
233
constexpr PR_CloseDelete() = default;
234
235
PR_CloseDelete(const PR_CloseDelete& aOther) = default;
236
237
void operator()(PRFileDesc* aPtr) const { PR_Close(aPtr); }
238
};
239
240
//
241
// Communication between the main thread and the writer thread.
242
//
243
// Main thread:
244
//
245
// * Whenever a shutdown step has been completed, the main thread
246
// obtains the number of ticks from the watchdog threads, builds
247
// a string representing all the data gathered so far, places
248
// this string in `gWriteData`, and wakes up the writer thread
249
// using `gWriteReady`. If `gWriteData` already contained a non-null
250
// pointer, this means that the writer thread is lagging behind the
251
// main thread, and the main thread cleans up the memory.
252
//
253
// Writer thread:
254
//
255
// * When awake, the writer thread swaps `gWriteData` to nullptr. If
256
// `gWriteData` contained data to write, the . If so, the writer
257
// thread writes the data to a file named "ShutdownDuration.json.tmp",
258
// then moves that file to "ShutdownDuration.json" and cleans up the
259
// data. If `gWriteData` contains a nullptr, the writer goes to sleep
260
// until it is awkened using `gWriteReady`.
261
//
262
//
263
// The data written by the writer thread will be read by another
264
// module upon the next restart and fed to Telemetry.
265
//
266
Atomic<nsCString*> gWriteData(nullptr);
267
PRMonitor* gWriteReady = nullptr;
268
269
void RunWriter(void* arg) {
270
AUTO_PROFILER_REGISTER_THREAD("Shutdown Statistics Writer");
271
NS_SetCurrentThreadName("Shutdown Statistics Writer");
272
273
MOZ_LSAN_INTENTIONALLY_LEAK_OBJECT(arg);
274
// Shutdown will generally complete before we have a chance to
275
// deallocate. This is not a leak.
276
277
// Setup destinationPath and tmpFilePath
278
279
nsCString destinationPath;
280
destinationPath.Adopt(static_cast<char*>(arg));
281
nsAutoCString tmpFilePath;
282
tmpFilePath.Append(destinationPath);
283
tmpFilePath.AppendLiteral(".tmp");
284
285
// Cleanup any file leftover from a previous run
286
Unused << PR_Delete(tmpFilePath.get());
287
Unused << PR_Delete(destinationPath.get());
288
289
while (true) {
290
//
291
// Check whether we have received data from the main thread.
292
//
293
// We perform the check before waiting on `gWriteReady` as we may
294
// have received data while we were busy writing.
295
//
296
// Also note that gWriteData may have been modified several times
297
// since we last checked. That's ok, we are not losing any important
298
// data (since we keep adding data), and we are not leaking memory
299
// (since the main thread deallocates any data that hasn't been
300
// consumed by the writer thread).
301
//
302
UniquePtr<nsCString> data(gWriteData.exchange(nullptr));
303
if (!data) {
304
// Data is not available yet.
305
// Wait until the main thread provides it.
306
PR_EnterMonitor(gWriteReady);
307
PR_Wait(gWriteReady, PR_INTERVAL_NO_TIMEOUT);
308
PR_ExitMonitor(gWriteReady);
309
continue;
310
}
311
312
MOZ_LSAN_INTENTIONALLY_LEAK_OBJECT(data.get());
313
// Shutdown may complete before we have a chance to deallocate.
314
// This is not a leak.
315
316
//
317
// Write to a temporary file
318
//
319
// In case of any error, we simply give up. Since the data is
320
// hardly critical, we don't want to spend too much effort
321
// salvaging it.
322
//
323
UniquePtr<PRFileDesc, PR_CloseDelete> tmpFileDesc(PR_Open(
324
tmpFilePath.get(), PR_WRONLY | PR_TRUNCATE | PR_CREATE_FILE, 00600));
325
326
// Shutdown may complete before we have a chance to close the file.
327
// This is not a leak.
328
MOZ_LSAN_INTENTIONALLY_LEAK_OBJECT(tmpFileDesc.get());
329
330
if (tmpFileDesc == nullptr) {
331
break;
332
}
333
if (PR_Write(tmpFileDesc.get(), data->get(), data->Length()) == -1) {
334
break;
335
}
336
tmpFileDesc.reset();
337
338
//
339
// Rename on top of destination file.
340
//
341
// This is not sufficient to guarantee that the destination file
342
// will be written correctly, but, again, we don't care enough
343
// about the data to make more efforts.
344
//
345
if (PR_Rename(tmpFilePath.get(), destinationPath.get()) != PR_SUCCESS) {
346
break;
347
}
348
}
349
}
350
351
} // namespace
352
353
NS_IMPL_ISUPPORTS(nsTerminator, nsIObserver)
354
355
nsTerminator::nsTerminator() : mInitialized(false), mCurrentStep(-1) {}
356
357
// During startup, register as an observer for all interesting topics.
358
nsresult nsTerminator::SelfInit() {
359
nsCOMPtr<nsIObserverService> os = mozilla::services::GetObserverService();
360
if (!os) {
361
return NS_ERROR_UNEXPECTED;
362
}
363
364
for (auto& shutdownStep : sShutdownSteps) {
365
DebugOnly<nsresult> rv = os->AddObserver(this, shutdownStep.mTopic, false);
366
NS_WARNING_ASSERTION(NS_SUCCEEDED(rv), "AddObserver failed");
367
}
368
369
return NS_OK;
370
}
371
372
// Actually launch these threads. This takes place at the first sign of
373
// shutdown.
374
void nsTerminator::Start() {
375
MOZ_ASSERT(!mInitialized);
376
StartWatchdog();
377
#if !defined(NS_FREE_PERMANENT_DATA)
378
// Only allow nsTerminator to write on non-leak-checked builds so we don't
379
// get leak warnings on shutdown for intentional leaks (see bug 1242084).
380
// This will be enabled again by bug 1255484 when 1255478 lands.
381
StartWriter();
382
#endif // !defined(NS_FREE_PERMANENT_DATA)
383
mInitialized = true;
384
sShutdownNotified = false;
385
}
386
387
// Prepare, allocate and start the watchdog thread.
388
// By design, it will never finish, nor be deallocated.
389
void nsTerminator::StartWatchdog() {
390
int32_t crashAfterMS =
391
Preferences::GetInt("toolkit.asyncshutdown.crash_timeout",
392
FALLBACK_ASYNCSHUTDOWN_CRASH_AFTER_MS);
393
// Ignore negative values
394
if (crashAfterMS <= 0) {
395
crashAfterMS = FALLBACK_ASYNCSHUTDOWN_CRASH_AFTER_MS;
396
}
397
398
// Add a little padding, to ensure that we do not crash before
399
// AsyncShutdown.
400
if (crashAfterMS > INT32_MAX - ADDITIONAL_WAIT_BEFORE_CRASH_MS) {
401
// Defend against overflow
402
crashAfterMS = INT32_MAX;
403
} else {
404
crashAfterMS += ADDITIONAL_WAIT_BEFORE_CRASH_MS;
405
}
406
407
#ifdef MOZ_VALGRIND
408
// If we're running on Valgrind, we'll be making forward progress at a
409
// rate of somewhere between 1/25th and 1/50th of normal. This can cause
410
// timeouts frequently enough to be a problem for the Valgrind runs on
411
// automation: see bug 1296819. As an attempt to avoid the worst of this,
412
// scale up the presented timeout by a factor of three. For a
413
// non-Valgrind-enabled build, or for an enabled build which isn't running
414
// on Valgrind, the timeout is unchanged.
415
if (RUNNING_ON_VALGRIND) {
416
const int32_t scaleUp = 3;
417
if (crashAfterMS >= (INT32_MAX / scaleUp) - 1) {
418
// Defend against overflow
419
crashAfterMS = INT32_MAX;
420
} else {
421
crashAfterMS *= scaleUp;
422
}
423
}
424
#endif
425
426
UniquePtr<Options> options(new Options());
427
const PRIntervalTime ticksDuration = PR_MillisecondsToInterval(1000);
428
options->crashAfterTicks = crashAfterMS / ticksDuration;
429
// Handle systems where ticksDuration is greater than crashAfterMS.
430
if (options->crashAfterTicks == 0) {
431
options->crashAfterTicks = crashAfterMS / 1000;
432
}
433
434
DebugOnly<PRThread*> watchdogThread =
435
CreateSystemThread(RunWatchdog, options.release());
436
MOZ_ASSERT(watchdogThread);
437
}
438
439
// Prepare, allocate and start the writer thread. By design, it will never
440
// finish, nor be deallocated. In case of error, we degrade
441
// gracefully to not writing Telemetry data.
442
void nsTerminator::StartWriter() {
443
if (!Telemetry::CanRecordExtended()) {
444
return;
445
}
446
nsCOMPtr<nsIFile> profLD;
447
nsresult rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_LOCAL_50_DIR,
448
getter_AddRefs(profLD));
449
if (NS_FAILED(rv)) {
450
return;
451
}
452
453
rv = profLD->Append(NS_LITERAL_STRING("ShutdownDuration.json"));
454
if (NS_FAILED(rv)) {
455
return;
456
}
457
458
nsAutoString path;
459
rv = profLD->GetPath(path);
460
if (NS_FAILED(rv)) {
461
return;
462
}
463
464
gWriteReady = PR_NewMonitor();
465
MOZ_LSAN_INTENTIONALLY_LEAK_OBJECT(
466
gWriteReady); // We will never deallocate this object
467
PRThread* writerThread = CreateSystemThread(RunWriter, ToNewUTF8String(path));
468
469
if (!writerThread) {
470
return;
471
}
472
}
473
474
NS_IMETHODIMP
475
nsTerminator::Observe(nsISupports*, const char* aTopic, const char16_t*) {
476
if (strcmp(aTopic, "profile-after-change") == 0) {
477
return SelfInit();
478
}
479
480
// Other notifications are shutdown-related.
481
482
// As we have seen examples in the wild of shutdown notifications
483
// not being sent (or not being sent in the expected order), we do
484
// not assume a specific order.
485
if (!mInitialized) {
486
Start();
487
}
488
489
UpdateHeartbeat(aTopic);
490
#if !defined(NS_FREE_PERMANENT_DATA)
491
// Only allow nsTerminator to write on non-leak checked builds so we don't get
492
// leak warnings on shutdown for intentional leaks (see bug 1242084). This
493
// will be enabled again by bug 1255484 when 1255478 lands.
494
UpdateTelemetry();
495
#endif // !defined(NS_FREE_PERMANENT_DATA)
496
UpdateCrashReport(aTopic);
497
498
// Perform a little cleanup
499
nsCOMPtr<nsIObserverService> os = mozilla::services::GetObserverService();
500
MOZ_RELEASE_ASSERT(os);
501
(void)os->RemoveObserver(this, aTopic);
502
503
return NS_OK;
504
}
505
506
void nsTerminator::UpdateHeartbeat(const char* aTopic) {
507
// Reset the clock, find out how long the current phase has lasted.
508
uint32_t ticks = gHeartbeat.exchange(0);
509
if (mCurrentStep > 0) {
510
sShutdownSteps[mCurrentStep].mTicks = ticks;
511
}
512
513
// Find out where we now are in the current shutdown.
514
// Don't assume that shutdown takes place in the expected order.
515
int nextStep = -1;
516
for (size_t i = 0; i < ArrayLength(sShutdownSteps); ++i) {
517
if (strcmp(sShutdownSteps[i].mTopic, aTopic) == 0) {
518
nextStep = i;
519
break;
520
}
521
}
522
MOZ_ASSERT(nextStep != -1);
523
mCurrentStep = nextStep;
524
}
525
526
void nsTerminator::UpdateTelemetry() {
527
if (!Telemetry::CanRecordExtended() || !gWriteReady) {
528
return;
529
}
530
531
//
532
// We need Telemetry data on the effective duration of each step,
533
// to be able to tune the time-to-crash of each of both the
534
// Terminator and AsyncShutdown. However, at this stage, it is too
535
// late to record such data into Telemetry, so we write it to disk
536
// and read it upon the next startup.
537
//
538
539
// Build JSON.
540
UniquePtr<nsCString> telemetryData(new nsCString());
541
telemetryData->AppendLiteral("{");
542
size_t fields = 0;
543
for (auto& shutdownStep : sShutdownSteps) {
544
if (shutdownStep.mTicks < 0) {
545
// Ignore this field.
546
continue;
547
}
548
if (fields++ > 0) {
549
telemetryData->AppendLiteral(", ");
550
}
551
telemetryData->AppendLiteral(R"(")");
552
telemetryData->Append(shutdownStep.mTopic);
553
telemetryData->AppendLiteral(R"(": )");
554
telemetryData->AppendInt(shutdownStep.mTicks);
555
}
556
telemetryData->AppendLiteral("}");
557
558
if (fields == 0) {
559
// Nothing to write
560
return;
561
}
562
563
//
564
// Send data to the worker thread.
565
//
566
delete gWriteData.exchange(
567
telemetryData.release()); // Clear any data that hasn't been written yet
568
569
// In case the worker thread was sleeping, wake it up.
570
PR_EnterMonitor(gWriteReady);
571
PR_Notify(gWriteReady);
572
PR_ExitMonitor(gWriteReady);
573
}
574
575
void nsTerminator::UpdateCrashReport(const char* aTopic) {
576
// In case of crash, we wish to know where in shutdown we are
577
nsAutoCString report(aTopic);
578
579
Unused << CrashReporter::AnnotateCrashReport(
580
CrashReporter::Annotation::ShutdownProgress, report);
581
}
582
583
void XPCOMShutdownNotified() {
584
MOZ_DIAGNOSTIC_ASSERT(sShutdownNotified == false);
585
sShutdownNotified = true;
586
}
587
588
} // namespace mozilla