Source code

Revision control

Other Tools

1
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2
* vim: set ts=8 sw=2 et tw=80:
3
*
4
* This Source Code Form is subject to the terms of the Mozilla Public
5
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
6
* You can obtain one at http://mozilla.org/MPL/2.0/. */
7
8
#include "gc/Nursery-inl.h"
9
10
#include "mozilla/DebugOnly.h"
11
#include "mozilla/IntegerPrintfMacros.h"
12
#include "mozilla/Move.h"
13
#include "mozilla/Unused.h"
14
15
#include "jsutil.h"
16
17
#include "builtin/MapObject.h"
18
#include "debugger/DebugAPI.h"
19
#include "gc/FreeOp.h"
20
#include "gc/GCInternals.h"
21
#include "gc/Memory.h"
22
#include "gc/PublicIterators.h"
23
#include "jit/JitFrames.h"
24
#include "jit/JitRealm.h"
25
#include "vm/ArrayObject.h"
26
#if defined(DEBUG)
27
# include "vm/EnvironmentObject.h"
28
#endif
29
#include "vm/JSONPrinter.h"
30
#include "vm/Realm.h"
31
#include "vm/Time.h"
32
#include "vm/TypedArrayObject.h"
33
#include "vm/TypeInference.h"
34
35
#include "gc/Marking-inl.h"
36
#include "gc/Zone-inl.h"
37
#include "vm/NativeObject-inl.h"
38
39
using namespace js;
40
using namespace gc;
41
42
using mozilla::DebugOnly;
43
using mozilla::PodCopy;
44
using mozilla::TimeDuration;
45
using mozilla::TimeStamp;
46
47
constexpr uintptr_t CanaryMagicValue = 0xDEADB15D;
48
49
#ifdef JS_GC_ZEAL
50
struct js::Nursery::Canary {
51
uintptr_t magicValue;
52
Canary* next;
53
};
54
#endif
55
56
namespace js {
57
struct NurseryChunk {
58
char data[Nursery::NurseryChunkUsableSize];
59
gc::ChunkTrailer trailer;
60
static NurseryChunk* fromChunk(gc::Chunk* chunk);
61
void poisonAndInit(JSRuntime* rt, size_t size = ChunkSize);
62
void poisonRange(size_t from, size_t size, uint8_t value,
63
MemCheckKind checkKind);
64
void poisonAfterEvict(size_t extent = ChunkSize);
65
66
// The end of the range is always ChunkSize - ArenaSize.
67
void markPagesUnusedHard(size_t from);
68
// The start of the range is always the beginning of the chunk.
69
MOZ_MUST_USE bool markPagesInUseHard(size_t to);
70
71
uintptr_t start() const { return uintptr_t(&data); }
72
uintptr_t end() const { return uintptr_t(&trailer); }
73
gc::Chunk* toChunk(JSRuntime* rt);
74
};
75
static_assert(sizeof(js::NurseryChunk) == gc::ChunkSize,
76
"Nursery chunk size must match gc::Chunk size.");
77
78
} // namespace js
79
80
inline void js::NurseryChunk::poisonAndInit(JSRuntime* rt, size_t size) {
81
poisonRange(0, size, JS_FRESH_NURSERY_PATTERN, MemCheckKind::MakeUndefined);
82
MOZ_MAKE_MEM_UNDEFINED(&trailer, sizeof(trailer));
83
new (&trailer) gc::ChunkTrailer(rt, &rt->gc.storeBuffer());
84
}
85
86
inline void js::NurseryChunk::poisonRange(size_t from, size_t size,
87
uint8_t value,
88
MemCheckKind checkKind) {
89
MOZ_ASSERT(from <= js::Nursery::NurseryChunkUsableSize);
90
MOZ_ASSERT(from + size <= ChunkSize);
91
92
uint8_t* start = reinterpret_cast<uint8_t*>(this) + from;
93
94
// We can poison the same chunk more than once, so first make sure memory
95
// sanitizers will let us poison it.
96
MOZ_MAKE_MEM_UNDEFINED(start, size);
97
Poison(start, value, size, checkKind);
98
}
99
100
inline void js::NurseryChunk::poisonAfterEvict(size_t extent) {
101
MOZ_ASSERT(extent <= ChunkSize);
102
poisonRange(0, extent, JS_SWEPT_NURSERY_PATTERN, MemCheckKind::MakeNoAccess);
103
}
104
105
inline void js::NurseryChunk::markPagesUnusedHard(size_t from) {
106
MOZ_ASSERT(from < ChunkSize - ArenaSize);
107
MarkPagesUnusedHard(reinterpret_cast<void*>(start() + from),
108
ChunkSize - ArenaSize - from);
109
}
110
111
inline bool js::NurseryChunk::markPagesInUseHard(size_t to) {
112
MOZ_ASSERT(to <= ChunkSize - ArenaSize);
113
return MarkPagesInUseHard(reinterpret_cast<void*>(start()), to);
114
}
115
116
// static
117
inline js::NurseryChunk* js::NurseryChunk::fromChunk(Chunk* chunk) {
118
return reinterpret_cast<NurseryChunk*>(chunk);
119
}
120
121
inline Chunk* js::NurseryChunk::toChunk(JSRuntime* rt) {
122
auto chunk = reinterpret_cast<Chunk*>(this);
123
chunk->init(rt);
124
return chunk;
125
}
126
127
void js::NurseryDecommitTask::queueChunk(
128
NurseryChunk* nchunk, const AutoLockHelperThreadState& lock) {
129
// Using the chunk pointers to build the queue is infallible.
130
Chunk* chunk = nchunk->toChunk(runtime());
131
chunk->info.prev = nullptr;
132
chunk->info.next = queue;
133
queue = chunk;
134
}
135
136
void js::NurseryDecommitTask::queueRange(
137
size_t newCapacity, NurseryChunk& newChunk,
138
const AutoLockHelperThreadState& lock) {
139
MOZ_ASSERT(!partialChunk || partialChunk == &newChunk);
140
141
// Only save this to decommit later if there's at least one page to
142
// decommit.
143
if (JS_ROUNDUP(newCapacity, SystemPageSize()) >=
144
JS_ROUNDDOWN(Nursery::NurseryChunkUsableSize, SystemPageSize())) {
145
// Clear the existing decommit request because it may be a larger request
146
// for the same chunk.
147
partialChunk = nullptr;
148
return;
149
}
150
partialChunk = &newChunk;
151
partialCapacity = newCapacity;
152
}
153
154
Chunk* js::NurseryDecommitTask::popChunk(
155
const AutoLockHelperThreadState& lock) {
156
if (!queue) {
157
return nullptr;
158
}
159
160
Chunk* chunk = queue;
161
queue = chunk->info.next;
162
chunk->info.next = nullptr;
163
MOZ_ASSERT(chunk->info.prev == nullptr);
164
return chunk;
165
}
166
167
void js::NurseryDecommitTask::run() {
168
Chunk* chunk;
169
170
{
171
AutoLockHelperThreadState lock;
172
173
while ((chunk = popChunk(lock)) || partialChunk) {
174
if (chunk) {
175
AutoUnlockHelperThreadState unlock(lock);
176
decommitChunk(chunk);
177
continue;
178
}
179
180
if (partialChunk) {
181
decommitRange(lock);
182
continue;
183
}
184
}
185
186
setFinishing(lock);
187
}
188
}
189
190
void js::NurseryDecommitTask::decommitChunk(Chunk* chunk) {
191
chunk->decommitAllArenas();
192
{
193
AutoLockGC lock(runtime());
194
runtime()->gc.recycleChunk(chunk, lock);
195
}
196
}
197
198
void js::NurseryDecommitTask::decommitRange(AutoLockHelperThreadState& lock) {
199
// Clear this field here before releasing the lock. While the lock is
200
// released the main thread may make new decommit requests or update the range
201
// of the current requested chunk, but it won't attempt to use any
202
// might-be-decommitted-soon memory.
203
NurseryChunk* thisPartialChunk = partialChunk;
204
size_t thisPartialCapacity = partialCapacity;
205
partialChunk = nullptr;
206
{
207
AutoUnlockHelperThreadState unlock(lock);
208
thisPartialChunk->markPagesUnusedHard(thisPartialCapacity);
209
}
210
}
211
212
js::Nursery::Nursery(JSRuntime* rt)
213
: runtime_(rt),
214
position_(0),
215
currentStartChunk_(0),
216
currentStartPosition_(0),
217
currentEnd_(0),
218
currentStringEnd_(0),
219
currentChunk_(0),
220
capacity_(0),
221
timeInChunkAlloc_(0),
222
profileThreshold_(0),
223
enableProfiling_(false),
224
canAllocateStrings_(true),
225
reportTenurings_(0),
226
minorGCTriggerReason_(JS::GCReason::NO_REASON),
227
decommitTask(rt)
228
#ifdef JS_GC_ZEAL
229
,
230
lastCanary_(nullptr)
231
#endif
232
{
233
const char* env = getenv("MOZ_NURSERY_STRINGS");
234
if (env && *env) {
235
canAllocateStrings_ = (*env == '1');
236
}
237
}
238
239
bool js::Nursery::init(AutoLockGCBgAlloc& lock) {
240
// The nursery is permanently disabled when recording or replaying. Nursery
241
// collections may occur at non-deterministic points in execution.
242
if (mozilla::recordreplay::IsRecordingOrReplaying()) {
243
return true;
244
}
245
246
capacity_ = roundSize(tunables().gcMinNurseryBytes());
247
if (!allocateNextChunk(0, lock)) {
248
capacity_ = 0;
249
return false;
250
}
251
// After this point the Nursery has been enabled.
252
253
setCurrentChunk(0);
254
setStartPosition();
255
poisonAndInitCurrentChunk();
256
257
char* env = getenv("JS_GC_PROFILE_NURSERY");
258
if (env) {
259
if (0 == strcmp(env, "help")) {
260
fprintf(stderr,
261
"JS_GC_PROFILE_NURSERY=N\n"
262
"\tReport minor GC's taking at least N microseconds.\n");
263
exit(0);
264
}
265
enableProfiling_ = true;
266
profileThreshold_ = TimeDuration::FromMicroseconds(atoi(env));
267
}
268
269
env = getenv("JS_GC_REPORT_TENURING");
270
if (env) {
271
if (0 == strcmp(env, "help")) {
272
fprintf(stderr,
273
"JS_GC_REPORT_TENURING=N\n"
274
"\tAfter a minor GC, report any ObjectGroups with at least N "
275
"instances tenured.\n");
276
exit(0);
277
}
278
reportTenurings_ = atoi(env);
279
}
280
281
if (!runtime()->gc.storeBuffer().enable()) {
282
return false;
283
}
284
285
MOZ_ASSERT(isEnabled());
286
return true;
287
}
288
289
js::Nursery::~Nursery() { disable(); }
290
291
void js::Nursery::enable() {
292
MOZ_ASSERT(isEmpty());
293
MOZ_ASSERT(!runtime()->gc.isVerifyPreBarriersEnabled());
294
if (isEnabled() || mozilla::recordreplay::IsRecordingOrReplaying()) {
295
return;
296
}
297
298
{
299
AutoLockGCBgAlloc lock(runtime());
300
capacity_ = roundSize(tunables().gcMinNurseryBytes());
301
if (!allocateNextChunk(0, lock)) {
302
capacity_ = 0;
303
return;
304
}
305
}
306
307
setCurrentChunk(0);
308
setStartPosition();
309
poisonAndInitCurrentChunk();
310
#ifdef JS_GC_ZEAL
311
if (runtime()->hasZealMode(ZealMode::GenerationalGC)) {
312
enterZealMode();
313
}
314
#endif
315
316
MOZ_ALWAYS_TRUE(runtime()->gc.storeBuffer().enable());
317
}
318
319
void js::Nursery::disable() {
320
MOZ_ASSERT(isEmpty());
321
if (!isEnabled()) {
322
return;
323
}
324
325
// Freeing the chunks must not race with decommitting part of one of our
326
// chunks. So join the decommitTask here and also below.
327
decommitTask.join();
328
freeChunksFrom(0);
329
capacity_ = 0;
330
331
// We must reset currentEnd_ so that there is no space for anything in the
332
// nursery. JIT'd code uses this even if the nursery is disabled.
333
currentEnd_ = 0;
334
currentStringEnd_ = 0;
335
position_ = 0;
336
runtime()->gc.storeBuffer().disable();
337
338
decommitTask.join();
339
}
340
341
void js::Nursery::enableStrings() {
342
MOZ_ASSERT(isEmpty());
343
canAllocateStrings_ = true;
344
currentStringEnd_ = currentEnd_;
345
}
346
347
void js::Nursery::disableStrings() {
348
MOZ_ASSERT(isEmpty());
349
canAllocateStrings_ = false;
350
currentStringEnd_ = 0;
351
}
352
353
bool js::Nursery::isEmpty() const {
354
if (!isEnabled()) {
355
return true;
356
}
357
358
if (!runtime()->hasZealMode(ZealMode::GenerationalGC)) {
359
MOZ_ASSERT(currentStartChunk_ == 0);
360
MOZ_ASSERT(currentStartPosition_ == chunk(0).start());
361
}
362
return position() == currentStartPosition_;
363
}
364
365
#ifdef JS_GC_ZEAL
366
void js::Nursery::enterZealMode() {
367
if (isEnabled()) {
368
if (isSubChunkMode()) {
369
// The poisoning call below must not race with background decommit,
370
// which could be attempting to decommit the currently-unused part of this
371
// chunk.
372
decommitTask.join();
373
{
374
AutoEnterOOMUnsafeRegion oomUnsafe;
375
if (!chunk(0).markPagesInUseHard(ChunkSize - ArenaSize)) {
376
oomUnsafe.crash("Out of memory trying to extend chunk for zeal mode");
377
}
378
}
379
380
// It'd be simpler to poison the whole chunk, but we can't do that
381
// because the nursery might be partially used.
382
chunk(0).poisonRange(capacity_, NurseryChunkUsableSize - capacity_,
383
JS_FRESH_NURSERY_PATTERN,
384
MemCheckKind::MakeUndefined);
385
}
386
capacity_ = JS_ROUNDUP(tunables().gcMaxNurseryBytes(), ChunkSize);
387
setCurrentEnd();
388
}
389
}
390
391
void js::Nursery::leaveZealMode() {
392
if (isEnabled()) {
393
MOZ_ASSERT(isEmpty());
394
setCurrentChunk(0);
395
setStartPosition();
396
poisonAndInitCurrentChunk();
397
}
398
}
399
#endif // JS_GC_ZEAL
400
401
JSObject* js::Nursery::allocateObject(JSContext* cx, size_t size,
402
size_t nDynamicSlots,
403
const JSClass* clasp) {
404
// Ensure there's enough space to replace the contents with a
405
// RelocationOverlay.
406
MOZ_ASSERT(size >= sizeof(RelocationOverlay));
407
408
// Sanity check the finalizer.
409
MOZ_ASSERT_IF(clasp->hasFinalize(),
410
CanNurseryAllocateFinalizedClass(clasp) || clasp->isProxy());
411
412
// Make the object allocation.
413
JSObject* obj = static_cast<JSObject*>(allocate(size));
414
if (!obj) {
415
return nullptr;
416
}
417
418
// If we want external slots, add them.
419
HeapSlot* slots = nullptr;
420
if (nDynamicSlots) {
421
MOZ_ASSERT(clasp->isNative());
422
slots = static_cast<HeapSlot*>(
423
allocateBuffer(cx->zone(), nDynamicSlots * sizeof(HeapSlot)));
424
if (!slots) {
425
// It is safe to leave the allocated object uninitialized, since we
426
// do not visit unallocated things in the nursery.
427
return nullptr;
428
}
429
}
430
431
// Store slots pointer directly in new object. If no dynamic slots were
432
// requested, caller must initialize slots_ field itself as needed. We
433
// don't know if the caller was a native object or not.
434
if (nDynamicSlots) {
435
static_cast<NativeObject*>(obj)->initSlots(slots);
436
}
437
438
gcTracer.traceNurseryAlloc(obj, size);
439
return obj;
440
}
441
442
Cell* js::Nursery::allocateString(Zone* zone, size_t size, AllocKind kind) {
443
// Ensure there's enough space to replace the contents with a
444
// RelocationOverlay.
445
MOZ_ASSERT(size >= sizeof(RelocationOverlay));
446
447
size_t allocSize =
448
JS_ROUNDUP(sizeof(StringLayout) - 1 + size, CellAlignBytes);
449
auto header = static_cast<StringLayout*>(allocate(allocSize));
450
if (!header) {
451
return nullptr;
452
}
453
header->zone = zone;
454
455
auto cell = reinterpret_cast<Cell*>(&header->cell);
456
gcTracer.traceNurseryAlloc(cell, kind);
457
return cell;
458
}
459
460
void* js::Nursery::allocate(size_t size) {
461
MOZ_ASSERT(isEnabled());
462
MOZ_ASSERT(!JS::RuntimeHeapIsBusy());
463
MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime()));
464
MOZ_ASSERT_IF(currentChunk_ == currentStartChunk_,
465
position() >= currentStartPosition_);
466
MOZ_ASSERT(position() % CellAlignBytes == 0);
467
MOZ_ASSERT(size % CellAlignBytes == 0);
468
469
#ifdef JS_GC_ZEAL
470
static const size_t CanarySize =
471
(sizeof(Nursery::Canary) + CellAlignBytes - 1) & ~CellAlignMask;
472
if (runtime()->gc.hasZealMode(ZealMode::CheckNursery)) {
473
size += CanarySize;
474
}
475
#endif
476
477
if (currentEnd() < position() + size) {
478
unsigned chunkno = currentChunk_ + 1;
479
MOZ_ASSERT(chunkno <= maxChunkCount());
480
MOZ_ASSERT(chunkno <= allocatedChunkCount());
481
if (chunkno == maxChunkCount()) {
482
return nullptr;
483
}
484
if (MOZ_UNLIKELY(chunkno == allocatedChunkCount())) {
485
mozilla::TimeStamp start = ReallyNow();
486
{
487
AutoLockGCBgAlloc lock(runtime());
488
if (!allocateNextChunk(chunkno, lock)) {
489
return nullptr;
490
}
491
}
492
timeInChunkAlloc_ += ReallyNow() - start;
493
MOZ_ASSERT(chunkno < allocatedChunkCount());
494
}
495
setCurrentChunk(chunkno);
496
poisonAndInitCurrentChunk();
497
}
498
499
void* thing = (void*)position();
500
position_ = position() + size;
501
// We count this regardless of the profiler's state, assuming that it costs
502
// just as much to count it, as to check the profiler's state and decide not
503
// to count it.
504
stats().noteNurseryAlloc();
505
506
DebugOnlyPoison(thing, JS_ALLOCATED_NURSERY_PATTERN, size,
507
MemCheckKind::MakeUndefined);
508
509
#ifdef JS_GC_ZEAL
510
if (runtime()->gc.hasZealMode(ZealMode::CheckNursery)) {
511
auto canary = reinterpret_cast<Canary*>(position() - CanarySize);
512
canary->magicValue = CanaryMagicValue;
513
canary->next = nullptr;
514
if (lastCanary_) {
515
MOZ_ASSERT(!lastCanary_->next);
516
lastCanary_->next = canary;
517
}
518
lastCanary_ = canary;
519
}
520
#endif
521
522
return thing;
523
}
524
525
void* js::Nursery::allocateBuffer(Zone* zone, size_t nbytes) {
526
MOZ_ASSERT(nbytes > 0);
527
528
if (nbytes <= MaxNurseryBufferSize) {
529
void* buffer = allocate(nbytes);
530
if (buffer) {
531
return buffer;
532
}
533
}
534
535
void* buffer = zone->pod_malloc<uint8_t>(nbytes);
536
if (buffer && !registerMallocedBuffer(buffer)) {
537
js_free(buffer);
538
return nullptr;
539
}
540
return buffer;
541
}
542
543
void* js::Nursery::allocateBuffer(JSObject* obj, size_t nbytes) {
544
MOZ_ASSERT(obj);
545
MOZ_ASSERT(nbytes > 0);
546
547
if (!IsInsideNursery(obj)) {
548
return obj->zone()->pod_malloc<uint8_t>(nbytes);
549
}
550
return allocateBuffer(obj->zone(), nbytes);
551
}
552
553
void* js::Nursery::allocateBufferSameLocation(JSObject* obj, size_t nbytes) {
554
MOZ_ASSERT(obj);
555
MOZ_ASSERT(nbytes > 0);
556
MOZ_ASSERT(nbytes <= MaxNurseryBufferSize);
557
558
if (!IsInsideNursery(obj)) {
559
return obj->zone()->pod_malloc<uint8_t>(nbytes);
560
}
561
562
return allocate(nbytes);
563
}
564
565
void* js::Nursery::allocateZeroedBuffer(
566
Zone* zone, size_t nbytes, arena_id_t arena /*= js::MallocArena*/) {
567
MOZ_ASSERT(nbytes > 0);
568
569
if (nbytes <= MaxNurseryBufferSize) {
570
void* buffer = allocate(nbytes);
571
if (buffer) {
572
memset(buffer, 0, nbytes);
573
return buffer;
574
}
575
}
576
577
void* buffer = zone->pod_calloc<uint8_t>(nbytes, arena);
578
if (buffer && !registerMallocedBuffer(buffer)) {
579
js_free(buffer);
580
return nullptr;
581
}
582
return buffer;
583
}
584
585
void* js::Nursery::allocateZeroedBuffer(
586
JSObject* obj, size_t nbytes, arena_id_t arena /*= js::MallocArena*/) {
587
MOZ_ASSERT(obj);
588
MOZ_ASSERT(nbytes > 0);
589
590
if (!IsInsideNursery(obj)) {
591
return obj->zone()->pod_calloc<uint8_t>(nbytes, arena);
592
}
593
return allocateZeroedBuffer(obj->zone(), nbytes, arena);
594
}
595
596
void* js::Nursery::reallocateBuffer(JSObject* obj, void* oldBuffer,
597
size_t oldBytes, size_t newBytes) {
598
if (!IsInsideNursery(obj)) {
599
return obj->zone()->pod_realloc<uint8_t>((uint8_t*)oldBuffer, oldBytes,
600
newBytes);
601
}
602
603
if (!isInside(oldBuffer)) {
604
void* newBuffer = obj->zone()->pod_realloc<uint8_t>((uint8_t*)oldBuffer,
605
oldBytes, newBytes);
606
if (newBuffer && oldBuffer != newBuffer) {
607
MOZ_ALWAYS_TRUE(mallocedBuffers.rekeyAs(oldBuffer, newBuffer, newBuffer));
608
}
609
return newBuffer;
610
}
611
612
// The nursery cannot make use of the returned slots data.
613
if (newBytes < oldBytes) {
614
return oldBuffer;
615
}
616
617
void* newBuffer = allocateBuffer(obj->zone(), newBytes);
618
if (newBuffer) {
619
PodCopy((uint8_t*)newBuffer, (uint8_t*)oldBuffer, oldBytes);
620
}
621
return newBuffer;
622
}
623
624
void js::Nursery::freeBuffer(void* buffer) {
625
if (!isInside(buffer)) {
626
removeMallocedBuffer(buffer);
627
js_free(buffer);
628
}
629
}
630
631
void Nursery::setIndirectForwardingPointer(void* oldData, void* newData) {
632
MOZ_ASSERT(isInside(oldData));
633
634
// Bug 1196210: If a zero-capacity header lands in the last 2 words of a
635
// jemalloc chunk abutting the start of a nursery chunk, the (invalid)
636
// newData pointer will appear to be "inside" the nursery.
637
MOZ_ASSERT(!isInside(newData) || (uintptr_t(newData) & ChunkMask) == 0);
638
639
AutoEnterOOMUnsafeRegion oomUnsafe;
640
#ifdef DEBUG
641
if (ForwardedBufferMap::Ptr p = forwardedBuffers.lookup(oldData)) {
642
MOZ_ASSERT(p->value() == newData);
643
}
644
#endif
645
if (!forwardedBuffers.put(oldData, newData)) {
646
oomUnsafe.crash("Nursery::setForwardingPointer");
647
}
648
}
649
650
#ifdef DEBUG
651
static bool IsWriteableAddress(void* ptr) {
652
volatile uint64_t* vPtr = reinterpret_cast<volatile uint64_t*>(ptr);
653
*vPtr = *vPtr;
654
return true;
655
}
656
#endif
657
658
void js::Nursery::forwardBufferPointer(HeapSlot** pSlotsElems) {
659
HeapSlot* old = *pSlotsElems;
660
661
if (!isInside(old)) {
662
return;
663
}
664
665
// The new location for this buffer is either stored inline with it or in
666
// the forwardedBuffers table.
667
do {
668
if (ForwardedBufferMap::Ptr p = forwardedBuffers.lookup(old)) {
669
*pSlotsElems = reinterpret_cast<HeapSlot*>(p->value());
670
break;
671
}
672
673
*pSlotsElems = *reinterpret_cast<HeapSlot**>(old);
674
} while (false);
675
676
MOZ_ASSERT(!isInside(*pSlotsElems));
677
MOZ_ASSERT(IsWriteableAddress(*pSlotsElems));
678
}
679
680
js::TenuringTracer::TenuringTracer(JSRuntime* rt, Nursery* nursery)
681
: JSTracer(rt, JSTracer::TracerKindTag::Tenuring, TraceWeakMapKeysValues),
682
nursery_(*nursery),
683
tenuredSize(0),
684
tenuredCells(0),
685
objHead(nullptr),
686
objTail(&objHead),
687
stringHead(nullptr),
688
stringTail(&stringHead) {}
689
690
inline float js::Nursery::calcPromotionRate(bool* validForTenuring) const {
691
float used = float(previousGC.nurseryUsedBytes);
692
float capacity = float(previousGC.nurseryCapacity);
693
float tenured = float(previousGC.tenuredBytes);
694
float rate;
695
696
if (previousGC.nurseryUsedBytes > 0) {
697
if (validForTenuring) {
698
// We can only use promotion rates if they're likely to be valid,
699
// they're only valid if the nursery was at least 90% full.
700
*validForTenuring = used > capacity * 0.9f;
701
}
702
rate = tenured / used;
703
} else {
704
if (validForTenuring) {
705
*validForTenuring = false;
706
}
707
rate = 0.0f;
708
}
709
710
return rate;
711
}
712
713
void js::Nursery::renderProfileJSON(JSONPrinter& json) const {
714
if (!isEnabled()) {
715
json.beginObject();
716
json.property("status", "nursery disabled");
717
json.endObject();
718
return;
719
}
720
721
if (previousGC.reason == JS::GCReason::NO_REASON) {
722
// If the nursery was empty when the last minorGC was requested, then
723
// no nursery collection will have been performed but JSON may still be
724
// requested. (And as a public API, this function should not crash in
725
// such a case.)
726
json.beginObject();
727
json.property("status", "nursery empty");
728
json.endObject();
729
return;
730
}
731
732
json.beginObject();
733
734
json.property("status", "complete");
735
736
json.property("reason", JS::ExplainGCReason(previousGC.reason));
737
json.property("bytes_tenured", previousGC.tenuredBytes);
738
json.property("cells_tenured", previousGC.tenuredCells);
739
json.property("strings_tenured",
740
stats().getStat(gcstats::STAT_STRINGS_TENURED));
741
json.property("bytes_used", previousGC.nurseryUsedBytes);
742
json.property("cur_capacity", previousGC.nurseryCapacity);
743
const size_t newCapacity = capacity();
744
if (newCapacity != previousGC.nurseryCapacity) {
745
json.property("new_capacity", newCapacity);
746
}
747
if (previousGC.nurseryCommitted != previousGC.nurseryCapacity) {
748
json.property("lazy_capacity", previousGC.nurseryCommitted);
749
}
750
if (!timeInChunkAlloc_.IsZero()) {
751
json.property("chunk_alloc_us", timeInChunkAlloc_, json.MICROSECONDS);
752
}
753
754
// These counters only contain consistent data if the profiler is enabled,
755
// and then there's no guarentee.
756
if (runtime()->geckoProfiler().enabled()) {
757
json.property("cells_allocated_nursery",
758
stats().allocsSinceMinorGCNursery());
759
json.property("cells_allocated_tenured",
760
stats().allocsSinceMinorGCTenured());
761
}
762
763
if (stats().getStat(gcstats::STAT_OBJECT_GROUPS_PRETENURED)) {
764
json.property("groups_pretenured",
765
stats().getStat(gcstats::STAT_OBJECT_GROUPS_PRETENURED));
766
}
767
if (stats().getStat(gcstats::STAT_NURSERY_STRING_REALMS_DISABLED)) {
768
json.property(
769
"nursery_string_realms_disabled",
770
stats().getStat(gcstats::STAT_NURSERY_STRING_REALMS_DISABLED));
771
}
772
773
json.beginObjectProperty("phase_times");
774
775
#define EXTRACT_NAME(name, text) #name,
776
static const char* const names[] = {
777
FOR_EACH_NURSERY_PROFILE_TIME(EXTRACT_NAME)
778
#undef EXTRACT_NAME
779
""};
780
781
size_t i = 0;
782
for (auto time : profileDurations_) {
783
json.property(names[i++], time, json.MICROSECONDS);
784
}
785
786
json.endObject(); // timings value
787
788
json.endObject();
789
}
790
791
// static
792
void js::Nursery::printProfileHeader() {
793
fprintf(stderr, "MinorGC: Reason PRate Size ");
794
#define PRINT_HEADER(name, text) fprintf(stderr, " %6s", text);
795
FOR_EACH_NURSERY_PROFILE_TIME(PRINT_HEADER)
796
#undef PRINT_HEADER
797
fprintf(stderr, "\n");
798
}
799
800
// static
801
void js::Nursery::printProfileDurations(const ProfileDurations& times) {
802
for (auto time : times) {
803
fprintf(stderr, " %6" PRIi64, static_cast<int64_t>(time.ToMicroseconds()));
804
}
805
fprintf(stderr, "\n");
806
}
807
808
void js::Nursery::printTotalProfileTimes() {
809
if (enableProfiling_) {
810
fprintf(stderr, "MinorGC TOTALS: %7" PRIu64 " collections: ",
811
runtime()->gc.minorGCCount());
812
printProfileDurations(totalDurations_);
813
}
814
}
815
816
void js::Nursery::maybeClearProfileDurations() {
817
for (auto& duration : profileDurations_) {
818
duration = mozilla::TimeDuration();
819
}
820
}
821
822
inline void js::Nursery::startProfile(ProfileKey key) {
823
startTimes_[key] = ReallyNow();
824
}
825
826
inline void js::Nursery::endProfile(ProfileKey key) {
827
profileDurations_[key] = ReallyNow() - startTimes_[key];
828
totalDurations_[key] += profileDurations_[key];
829
}
830
831
bool js::Nursery::shouldCollect() const {
832
if (isEmpty()) {
833
return false;
834
}
835
836
if (minorGCRequested()) {
837
return true;
838
}
839
840
bool belowBytesThreshold =
841
freeSpace() < tunables().nurseryFreeThresholdForIdleCollection();
842
bool belowFractionThreshold =
843
float(freeSpace()) / float(capacity()) <
844
tunables().nurseryFreeThresholdForIdleCollectionFraction();
845
846
// We want to use belowBytesThreshold when the nursery is sufficiently large,
847
// and belowFractionThreshold when it's small.
848
//
849
// When the nursery is small then belowBytesThreshold is a lower threshold
850
// (triggered earlier) than belowFractionThreshold. So if the fraction
851
// threshold is true, the bytes one will be true also. The opposite is true
852
// when the nursery is large.
853
//
854
// Therefore, by the time we cross the threshold we care about, we've already
855
// crossed the other one, and we can boolean AND to use either condition
856
// without encoding any "is the nursery big/small" test/threshold. The point
857
// at which they cross is when the nursery is: BytesThreshold /
858
// FractionThreshold large.
859
//
860
// With defaults that's:
861
//
862
// 1MB = 256KB / 0.25
863
//
864
return belowBytesThreshold && belowFractionThreshold;
865
}
866
867
// typeReason is the gcReason for specified type, for example,
868
// FULL_CELL_PTR_OBJ_BUFFER is the gcReason for JSObject.
869
static inline bool IsFullStoreBufferReason(JS::GCReason reason,
870
JS::GCReason typeReason) {
871
return reason == typeReason ||
872
reason == JS::GCReason::FULL_WHOLE_CELL_BUFFER ||
873
reason == JS::GCReason::FULL_GENERIC_BUFFER ||
874
reason == JS::GCReason::FULL_VALUE_BUFFER ||
875
reason == JS::GCReason::FULL_SLOT_BUFFER ||
876
reason == JS::GCReason::FULL_SHAPE_BUFFER;
877
}
878
879
void js::Nursery::collect(JS::GCReason reason) {
880
JSRuntime* rt = runtime();
881
MOZ_ASSERT(!rt->mainContextFromOwnThread()->suppressGC);
882
883
mozilla::recordreplay::AutoDisallowThreadEvents disallow;
884
885
if (!isEnabled() || isEmpty()) {
886
// Our barriers are not always exact, and there may be entries in the
887
// storebuffer even when the nursery is disabled or empty. It's not safe
888
// to keep these entries as they may refer to tenured cells which may be
889
// freed after this point.
890
rt->gc.storeBuffer().clear();
891
}
892
893
if (!isEnabled()) {
894
return;
895
}
896
897
#ifdef JS_GC_ZEAL
898
if (rt->gc.hasZealMode(ZealMode::CheckNursery)) {
899
for (auto canary = lastCanary_; canary; canary = canary->next) {
900
MOZ_ASSERT(canary->magicValue == CanaryMagicValue);
901
}
902
}
903
lastCanary_ = nullptr;
904
#endif
905
906
stats().beginNurseryCollection(reason);
907
gcTracer.traceMinorGCStart();
908
909
maybeClearProfileDurations();
910
startProfile(ProfileKey::Total);
911
912
// The analysis marks TenureCount as not problematic for GC hazards because
913
// it is only used here, and ObjectGroup pointers are never
914
// nursery-allocated.
915
MOZ_ASSERT(!IsNurseryAllocable(AllocKind::OBJECT_GROUP));
916
917
TenureCountCache tenureCounts;
918
previousGC.reason = JS::GCReason::NO_REASON;
919
if (!isEmpty()) {
920
doCollection(reason, tenureCounts);
921
} else {
922
previousGC.nurseryUsedBytes = 0;
923
previousGC.nurseryCapacity = capacity();
924
previousGC.nurseryCommitted = committed();
925
previousGC.tenuredBytes = 0;
926
previousGC.tenuredCells = 0;
927
}
928
929
// Resize the nursery.
930
maybeResizeNursery(reason);
931
932
// Poison/initialise the first chunk.
933
if (isEnabled() && previousGC.nurseryUsedBytes) {
934
// In most cases Nursery::clear() has not poisoned this chunk or marked it
935
// as NoAccess; so we only need to poison the region used during the last
936
// cycle. Also, if the heap was recently expanded we don't want to
937
// re-poison the new memory. In both cases we only need to poison until
938
// previousGC.nurseryUsedBytes.
939
//
940
// In cases where this is not true, like generational zeal mode or subchunk
941
// mode, poisonAndInitCurrentChunk() will ignore its parameter. It will
942
// also clamp the parameter.
943
poisonAndInitCurrentChunk(previousGC.nurseryUsedBytes);
944
}
945
946
const float promotionRate = doPretenuring(rt, reason, tenureCounts);
947
948
// We ignore gcMaxBytes when allocating for minor collection. However, if we
949
// overflowed, we disable the nursery. The next time we allocate, we'll fail
950
// because bytes >= gcMaxBytes.
951
if (rt->gc.heapSize.bytes() >= tunables().gcMaxBytes()) {
952
disable();
953
}
954
955
endProfile(ProfileKey::Total);
956
rt->gc.incMinorGcNumber();
957
958
TimeDuration totalTime = profileDurations_[ProfileKey::Total];
959
rt->addTelemetry(JS_TELEMETRY_GC_MINOR_US, totalTime.ToMicroseconds());
960
rt->addTelemetry(JS_TELEMETRY_GC_MINOR_REASON, uint32_t(reason));
961
if (totalTime.ToMilliseconds() > 1.0) {
962
rt->addTelemetry(JS_TELEMETRY_GC_MINOR_REASON_LONG, uint32_t(reason));
963
}
964
rt->addTelemetry(JS_TELEMETRY_GC_NURSERY_BYTES, committed());
965
966
stats().endNurseryCollection(reason);
967
gcTracer.traceMinorGCEnd();
968
timeInChunkAlloc_ = mozilla::TimeDuration();
969
970
if (enableProfiling_ && totalTime >= profileThreshold_) {
971
stats().maybePrintProfileHeaders();
972
973
fprintf(stderr, "MinorGC: %20s %5.1f%% %5zu ",
974
JS::ExplainGCReason(reason), promotionRate * 100,
975
capacity() / 1024);
976
printProfileDurations(profileDurations_);
977
978
if (reportTenurings_) {
979
for (auto& entry : tenureCounts.entries) {
980
if (entry.count >= reportTenurings_) {
981
fprintf(stderr, " %d x ", entry.count);
982
AutoSweepObjectGroup sweep(entry.group);
983
entry.group->print(sweep);
984
}
985
}
986
}
987
}
988
}
989
990
void js::Nursery::doCollection(JS::GCReason reason,
991
TenureCountCache& tenureCounts) {
992
JSRuntime* rt = runtime();
993
AutoGCSession session(rt, JS::HeapState::MinorCollecting);
994
AutoSetThreadIsPerformingGC performingGC;
995
AutoStopVerifyingBarriers av(rt, false);
996
AutoDisableProxyCheck disableStrictProxyChecking;
997
mozilla::DebugOnly<AutoEnterOOMUnsafeRegion> oomUnsafeRegion;
998
999
const size_t initialNurseryCapacity = capacity();
1000
const size_t initialNurseryUsedBytes = usedSpace();
1001
1002
// Move objects pointed to by roots from the nursery to the major heap.
1003
TenuringTracer mover(rt, this);
1004
1005
// Mark the store buffer. This must happen first.
1006
StoreBuffer& sb = runtime()->gc.storeBuffer();
1007
1008
// The MIR graph only contains nursery pointers if cancelIonCompilations()
1009
// is set on the store buffer, in which case we cancel all compilations
1010
// of such graphs.
1011
startProfile(ProfileKey::CancelIonCompilations);
1012
if (sb.cancelIonCompilations()) {
1013
js::CancelOffThreadIonCompilesUsingNurseryPointers(rt);
1014
}
1015
endProfile(ProfileKey::CancelIonCompilations);
1016
1017
startProfile(ProfileKey::TraceValues);
1018
sb.traceValues(mover);
1019
endProfile(ProfileKey::TraceValues);
1020
1021
startProfile(ProfileKey::TraceCells);
1022
sb.traceCells(mover);
1023
endProfile(ProfileKey::TraceCells);
1024
1025
startProfile(ProfileKey::TraceSlots);
1026
sb.traceSlots(mover);
1027
endProfile(ProfileKey::TraceSlots);
1028
1029
startProfile(ProfileKey::TraceWholeCells);
1030
sb.traceWholeCells(mover);
1031
endProfile(ProfileKey::TraceWholeCells);
1032
1033
startProfile(ProfileKey::TraceGenericEntries);
1034
sb.traceGenericEntries(&mover);
1035
endProfile(ProfileKey::TraceGenericEntries);
1036
1037
startProfile(ProfileKey::MarkRuntime);
1038
rt->gc.traceRuntimeForMinorGC(&mover, session);
1039
endProfile(ProfileKey::MarkRuntime);
1040
1041
startProfile(ProfileKey::MarkDebugger);
1042
{
1043
gcstats::AutoPhase ap(stats(), gcstats::PhaseKind::MARK_ROOTS);
1044
DebugAPI::traceAllForMovingGC(&mover);
1045
}
1046
endProfile(ProfileKey::MarkDebugger);
1047
1048
startProfile(ProfileKey::SweepCaches);
1049
rt->gc.purgeRuntimeForMinorGC();
1050
endProfile(ProfileKey::SweepCaches);
1051
1052
// Most of the work is done here. This loop iterates over objects that have
1053
// been moved to the major heap. If these objects have any outgoing pointers
1054
// to the nursery, then those nursery objects get moved as well, until no
1055
// objects are left to move. That is, we iterate to a fixed point.
1056
startProfile(ProfileKey::CollectToFP);
1057
collectToFixedPoint(mover, tenureCounts);
1058
endProfile(ProfileKey::CollectToFP);
1059
1060
// Sweep to update any pointers to nursery objects that have now been
1061
// tenured.
1062
startProfile(ProfileKey::Sweep);
1063
sweep(&mover);
1064
endProfile(ProfileKey::Sweep);
1065
1066
// Update any slot or element pointers whose destination has been tenured.
1067
startProfile(ProfileKey::UpdateJitActivations);
1068
js::jit::UpdateJitActivationsForMinorGC(rt);
1069
forwardedBuffers.clearAndCompact();
1070
endProfile(ProfileKey::UpdateJitActivations);
1071
1072
startProfile(ProfileKey::ObjectsTenuredCallback);
1073
rt->gc.callObjectsTenuredCallback();
1074
endProfile(ProfileKey::ObjectsTenuredCallback);
1075
1076
// Sweep.
1077
startProfile(ProfileKey::FreeMallocedBuffers);
1078
rt->gc.queueBuffersForFreeAfterMinorGC(mallocedBuffers);
1079
endProfile(ProfileKey::FreeMallocedBuffers);
1080
1081
startProfile(ProfileKey::ClearNursery);
1082
clear();
1083
endProfile(ProfileKey::ClearNursery);
1084
1085
startProfile(ProfileKey::ClearStoreBuffer);
1086
runtime()->gc.storeBuffer().clear();
1087
endProfile(ProfileKey::ClearStoreBuffer);
1088
1089
// Make sure hashtables have been updated after the collection.
1090
startProfile(ProfileKey::CheckHashTables);
1091
#ifdef JS_GC_ZEAL
1092
if (rt->hasZealMode(ZealMode::CheckHashTablesOnMinorGC)) {
1093
CheckHashTablesAfterMovingGC(rt);
1094
}
1095
#endif
1096
endProfile(ProfileKey::CheckHashTables);
1097
1098
previousGC.reason = reason;
1099
previousGC.nurseryCapacity = initialNurseryCapacity;
1100
previousGC.nurseryCommitted = spaceToEnd(allocatedChunkCount());
1101
previousGC.nurseryUsedBytes = initialNurseryUsedBytes;
1102
previousGC.tenuredBytes = mover.tenuredSize;
1103
previousGC.tenuredCells = mover.tenuredCells;
1104
}
1105
1106
float js::Nursery::doPretenuring(JSRuntime* rt, JS::GCReason reason,
1107
TenureCountCache& tenureCounts) {
1108
// If we are promoting the nursery, or exhausted the store buffer with
1109
// pointers to nursery things, which will force a collection well before
1110
// the nursery is full, look for object groups that are getting promoted
1111
// excessively and try to pretenure them.
1112
startProfile(ProfileKey::Pretenure);
1113
bool validPromotionRate;
1114
const float promotionRate = calcPromotionRate(&validPromotionRate);
1115
uint32_t pretenureCount = 0;
1116
bool attempt = tunables().attemptPretenuring();
1117
1118
bool pretenureObj, pretenureStr;
1119
if (attempt) {
1120
// Should we do pretenuring regardless of gcreason?
1121
bool shouldPretenure = validPromotionRate &&
1122
promotionRate > tunables().pretenureThreshold() &&
1123
previousGC.nurseryUsedBytes >= 4 * 1024 * 1024;
1124
pretenureObj =
1125
shouldPretenure ||
1126
IsFullStoreBufferReason(reason, JS::GCReason::FULL_CELL_PTR_OBJ_BUFFER);
1127
pretenureStr =
1128
shouldPretenure ||
1129
IsFullStoreBufferReason(reason, JS::GCReason::FULL_CELL_PTR_STR_BUFFER);
1130
} else {
1131
pretenureObj = false;
1132
pretenureStr = false;
1133
}
1134
1135
if (pretenureObj) {
1136
JSContext* cx = rt->mainContextFromOwnThread();
1137
uint32_t threshold = tunables().pretenureGroupThreshold();
1138
for (auto& entry : tenureCounts.entries) {
1139
if (entry.count < threshold) {
1140
continue;
1141
}
1142
1143
ObjectGroup* group = entry.group;
1144
AutoRealm ar(cx, group);
1145
AutoSweepObjectGroup sweep(group);
1146
if (group->canPreTenure(sweep)) {
1147
group->setShouldPreTenure(sweep, cx);
1148
pretenureCount++;
1149
}
1150
}
1151
}
1152
stats().setStat(gcstats::STAT_OBJECT_GROUPS_PRETENURED, pretenureCount);
1153
1154
mozilla::Maybe<AutoGCSession> session;
1155
uint32_t numStringsTenured = 0;
1156
uint32_t numNurseryStringRealmsDisabled = 0;
1157
for (ZonesIter zone(rt, SkipAtoms); !zone.done(); zone.next()) {
1158
if (pretenureStr && zone->allocNurseryStrings &&
1159
zone->tenuredStrings >= 30 * 1000) {
1160
if (!session.isSome()) {
1161
session.emplace(rt, JS::HeapState::MinorCollecting);
1162
}
1163
CancelOffThreadIonCompile(zone);
1164
bool preserving = zone->isPreservingCode();
1165
zone->setPreservingCode(false);
1166
zone->discardJitCode(rt->defaultFreeOp());
1167
zone->setPreservingCode(preserving);
1168
for (RealmsInZoneIter r(zone); !r.done(); r.next()) {
1169
if (jit::JitRealm* jitRealm = r->jitRealm()) {
1170
jitRealm->discardStubs();
1171
jitRealm->setStringsCanBeInNursery(false);
1172
numNurseryStringRealmsDisabled++;
1173
}
1174
}
1175
zone->allocNurseryStrings = false;
1176
}
1177
numStringsTenured += zone->tenuredStrings;
1178
zone->tenuredStrings = 0;
1179
}
1180
session.reset(); // End the minor GC session, if running one.
1181
stats().setStat(gcstats::STAT_NURSERY_STRING_REALMS_DISABLED,
1182
numNurseryStringRealmsDisabled);
1183
stats().setStat(gcstats::STAT_STRINGS_TENURED, numStringsTenured);
1184
endProfile(ProfileKey::Pretenure);
1185
1186
rt->addTelemetry(JS_TELEMETRY_GC_PRETENURE_COUNT, pretenureCount);
1187
rt->addTelemetry(JS_TELEMETRY_GC_NURSERY_PROMOTION_RATE, promotionRate * 100);
1188
1189
return promotionRate;
1190
}
1191
1192
bool js::Nursery::registerMallocedBuffer(void* buffer) {
1193
MOZ_ASSERT(buffer);
1194
return mallocedBuffers.putNew(buffer);
1195
}
1196
1197
void js::Nursery::sweep(JSTracer* trc) {
1198
// Sweep unique IDs first before we sweep any tables that may be keyed based
1199
// on them.
1200
for (Cell* cell : cellsWithUid_) {
1201
JSObject* obj = static_cast<JSObject*>(cell);
1202
if (!IsForwarded(obj)) {
1203
obj->zone()->removeUniqueId(obj);
1204
} else {
1205
JSObject* dst = Forwarded(obj);
1206
dst->zone()->transferUniqueId(dst, obj);
1207
}
1208
}
1209
cellsWithUid_.clear();
1210
1211
for (CompartmentsIter c(runtime()); !c.done(); c.next()) {
1212
c->sweepAfterMinorGC(trc);
1213
}
1214
1215
for (ZonesIter zone(trc->runtime(), SkipAtoms); !zone.done(); zone.next()) {
1216
zone->sweepAfterMinorGC(trc);
1217
}
1218
1219
sweepDictionaryModeObjects();
1220
sweepMapAndSetObjects();
1221
}
1222
1223
void js::Nursery::clear() {
1224
// Poison the nursery contents so touching a freed object will crash.
1225
unsigned firstClearChunk;
1226
if (runtime()->hasZealMode(ZealMode::GenerationalGC)) {
1227
// Poison all the chunks used in this cycle. The new start chunk is
1228
// reposioned in Nursery::collect() but there's no point optimising that in
1229
// this case.
1230
firstClearChunk = currentStartChunk_;
1231
} else {
1232
// In normal mode we start at the second chunk, the first one will be used
1233
// in the next cycle and poisoned in Nusery::collect();
1234
MOZ_ASSERT(currentStartChunk_ == 0);
1235
firstClearChunk = 1;
1236
}
1237
for (unsigned i = firstClearChunk; i < currentChunk_; ++i) {
1238
chunk(i).poisonAfterEvict();
1239
}
1240
// Clear only the used part of the chunk because that's the part we touched,
1241
// but only if it's not going to be re-used immediately (>= firstClearChunk).
1242
if (currentChunk_ >= firstClearChunk) {
1243
chunk(currentChunk_)
1244
.poisonAfterEvict(position() - chunk(currentChunk_).start());
1245
}
1246
1247
// Reset the start chunk & position if we're not in this zeal mode, or we're
1248
// in it and close to the end of the nursery.
1249
MOZ_ASSERT(maxChunkCount() > 0);
1250
if (!runtime()->hasZealMode(ZealMode::GenerationalGC) ||
1251
(runtime()->hasZealMode(ZealMode::GenerationalGC) &&
1252
currentChunk_ + 1 == maxChunkCount())) {
1253
setCurrentChunk(0);
1254
}
1255
1256
// Set current start position for isEmpty checks.
1257
setStartPosition();
1258
}
1259
1260
size_t js::Nursery::spaceToEnd(unsigned chunkCount) const {
1261
if (chunkCount == 0) {
1262
return 0;
1263
}
1264
1265
unsigned lastChunk = chunkCount - 1;
1266
1267
MOZ_ASSERT(lastChunk >= currentStartChunk_);
1268
MOZ_ASSERT(currentStartPosition_ - chunk(currentStartChunk_).start() <=
1269
NurseryChunkUsableSize);
1270
1271
size_t bytes;
1272
1273
if (chunkCount != 1) {
1274
// In the general case we have to add:
1275
// + the bytes used in the first
1276
// chunk which may be less than the total size of a chunk since in some
1277
// zeal modes we start the first chunk at some later position
1278
// (currentStartPosition_).
1279
// + the size of all the other chunks.
1280
bytes = (chunk(currentStartChunk_).end() - currentStartPosition_) +
1281
((lastChunk - currentStartChunk_) * ChunkSize);
1282
} else {
1283
// In sub-chunk mode, but it also works whenever chunkCount == 1, we need to
1284
// use currentEnd_ since it may not refer to a full chunk.
1285
bytes = currentEnd_ - currentStartPosition_;
1286
}
1287
1288
MOZ_ASSERT(bytes <= maxChunkCount() * ChunkSize);
1289
1290
return bytes;
1291
}
1292
1293
MOZ_ALWAYS_INLINE void js::Nursery::setCurrentChunk(unsigned chunkno) {
1294
MOZ_ASSERT(chunkno < allocatedChunkCount());
1295
1296
currentChunk_ = chunkno;
1297
position_ = chunk(chunkno).start();
1298
setCurrentEnd();
1299
}
1300
1301
void js::Nursery::poisonAndInitCurrentChunk(size_t extent) {
1302
if (runtime()->hasZealMode(ZealMode::GenerationalGC) || !isSubChunkMode()) {
1303
chunk(currentChunk_).poisonAndInit(runtime());
1304
} else {
1305
extent = Min(capacity_, extent);
1306
MOZ_ASSERT(extent <= NurseryChunkUsableSize);
1307
chunk(currentChunk_).poisonAndInit(runtime(), extent);
1308
}
1309
}
1310
1311
MOZ_ALWAYS_INLINE void js::Nursery::setCurrentEnd() {
1312
MOZ_ASSERT_IF(isSubChunkMode(),
1313
currentChunk_ == 0 && currentEnd_ <= chunk(0).end());
1314
currentEnd_ =
1315
chunk(currentChunk_).start() + Min(capacity_, NurseryChunkUsableSize);
1316
if (canAllocateStrings_) {
1317
currentStringEnd_ = currentEnd_;
1318
}
1319
}
1320
1321
bool js::Nursery::allocateNextChunk(const unsigned chunkno,
1322
AutoLockGCBgAlloc& lock) {
1323
const unsigned priorCount = allocatedChunkCount();
1324
const unsigned newCount = priorCount + 1;
1325
1326
MOZ_ASSERT((chunkno == currentChunk_ + 1) ||
1327
(chunkno == 0 && allocatedChunkCount() == 0));
1328
MOZ_ASSERT(chunkno == allocatedChunkCount());
1329
MOZ_ASSERT(chunkno < JS_HOWMANY(capacity(), ChunkSize));
1330
1331
if (!chunks_.resize(newCount)) {
1332
return false;
1333
}
1334
1335
Chunk* newChunk;
1336
newChunk = runtime()->gc.getOrAllocChunk(lock);
1337
if (!newChunk) {
1338
chunks_.shrinkTo(priorCount);
1339
return false;
1340
}
1341
1342
chunks_[chunkno] = NurseryChunk::fromChunk(newChunk);
1343
return true;
1344
}
1345
1346
MOZ_ALWAYS_INLINE void js::Nursery::setStartPosition() {
1347
currentStartChunk_ = currentChunk_;
1348
currentStartPosition_ = position();
1349
}
1350
1351
void js::Nursery::maybeResizeNursery(JS::GCReason reason) {
1352
if (maybeResizeExact(reason)) {
1353
return;
1354
}
1355
1356
// This incorrect promotion rate results in better nursery sizing
1357
// decisions, however we should to better tuning based on the real
1358
// promotion rate in the future.
1359
const float promotionRate =
1360
float(previousGC.tenuredBytes) / float(previousGC.nurseryCapacity);
1361
1362
// Object lifetimes aren't going to behave linearly, but a better
1363
// relationship that works for all programs and can be predicted in
1364
// advance doesn't exist.
1365
static const float GrowThreshold = 0.03f;
1366
static const float ShrinkThreshold = 0.01f;
1367
static const float PromotionGoal = (GrowThreshold + ShrinkThreshold) / 2.0f;
1368
const float factor = promotionRate / PromotionGoal;
1369
MOZ_ASSERT(factor >= 0.0f);
1370
1371
#ifdef DEBUG
1372
// This is |... <= SIZE_MAX|, just without the implicit value-changing
1373
// conversion that expression would involve and modern clang would warn about.
1374
static const float SizeMaxPlusOne =
1375
2.0f * float(1ULL << (sizeof(void*) * CHAR_BIT - 1));
1376
MOZ_ASSERT((float(capacity()) * factor) < SizeMaxPlusOne);
1377
#endif
1378
1379
size_t newCapacity = size_t(float(capacity()) * factor);
1380
1381
const size_t minNurseryBytes = roundSize(tunables().gcMinNurseryBytes());
1382
MOZ_ASSERT(minNurseryBytes >= ArenaSize);
1383
const size_t maxNurseryBytes = roundSize(tunables().gcMaxNurseryBytes());
1384
MOZ_ASSERT(maxNurseryBytes >= ArenaSize);
1385
1386
// If one of these conditions is true then we always shrink or grow the
1387
// nursery. This way the thresholds still have an effect even if the goal
1388
// seeking says the current size is ideal.
1389
size_t lowLimit = Max(minNurseryBytes, capacity() / 2);
1390
size_t highLimit =
1391
Min(maxNurseryBytes, (CheckedInt<size_t>(capacity()) * 2).value());
1392
newCapacity = roundSize(mozilla::Clamp(newCapacity, lowLimit, highLimit));
1393
1394
if (capacity() < maxNurseryBytes && promotionRate > GrowThreshold &&
1395
newCapacity > capacity()) {
1396
growAllocableSpace(newCapacity);
1397
} else if (capacity() >= minNurseryBytes + SubChunkStep &&
1398
promotionRate < ShrinkThreshold && newCapacity < capacity()) {
1399
shrinkAllocableSpace(newCapacity);
1400
}
1401
}
1402
1403
bool js::Nursery::maybeResizeExact(JS::GCReason reason) {
1404
// Shrink the nursery to its minimum size if we ran out of memory or
1405
// received a memory pressure event.
1406
if (gc::IsOOMReason(reason) || runtime()->gc.systemHasLowMemory()) {
1407
minimizeAllocableSpace();
1408
return true;
1409
}
1410
1411
#ifdef JS_GC_ZEAL
1412
// This zeal mode disabled nursery resizing.
1413
if (runtime()->hasZealMode(ZealMode::GenerationalGC)) {
1414
return true;
1415
}
1416
#endif
1417
1418
MOZ_ASSERT(tunables().gcMaxNurseryBytes() >= ArenaSize);
1419
const size_t newMaxNurseryBytes = roundSize(tunables().gcMaxNurseryBytes());
1420
MOZ_ASSERT(newMaxNurseryBytes >= ArenaSize);
1421
1422
if (capacity_ > newMaxNurseryBytes) {
1423
// The configured maximum nursery size is changing.
1424
// We need to shrink the nursery.
1425
shrinkAllocableSpace(newMaxNurseryBytes);
1426
return true;
1427
}
1428
1429
const size_t newMinNurseryBytes = roundSize(tunables().gcMinNurseryBytes());
1430
MOZ_ASSERT(newMinNurseryBytes >= ArenaSize);
1431
1432
if (newMinNurseryBytes > capacity()) {
1433
// the configured minimum nursery size is changing, so grow the nursery.
1434
MOZ_ASSERT(newMinNurseryBytes <= roundSize(tunables().gcMaxNurseryBytes()));
1435
growAllocableSpace(newMinNurseryBytes);
1436
return true;
1437
}
1438
1439
return false;
1440
}
1441
1442
size_t js::Nursery::roundSize(size_t size) {
1443
if (size >= ChunkSize) {
1444
size = JS_ROUND(size, ChunkSize);
1445
} else {
1446
size = Min(JS_ROUND(size, SubChunkStep),
1447
JS_ROUNDDOWN(NurseryChunkUsableSize, SubChunkStep));
1448
}
1449
MOZ_ASSERT(size >= ArenaSize);
1450
return size;
1451
}
1452
1453
void js::Nursery::growAllocableSpace(size_t newCapacity) {
1454
MOZ_ASSERT_IF(!isSubChunkMode(), newCapacity > currentChunk_ * ChunkSize);
1455
MOZ_ASSERT(newCapacity <= roundSize(tunables().gcMaxNurseryBytes()));
1456
MOZ_ASSERT(newCapacity > capacity());
1457
1458
if (isSubChunkMode()) {
1459
// Avoid growing into an area that's about to be decommitted.
1460
decommitTask.join();
1461
1462
MOZ_ASSERT(currentChunk_ == 0);
1463
1464
// The remainder of the chunk may have been decommitted.
1465
if (!chunk(0).markPagesInUseHard(Min(newCapacity, ChunkSize - ArenaSize))) {
1466
// The OS won't give us the memory we need, we can't grow.
1467
return;
1468
}
1469
1470
// The capacity has changed and since we were in sub-chunk mode we need to
1471
// update the poison values / asan infomation for the now-valid region of
1472
// this chunk.
1473
size_t poisonSize = Min(newCapacity, NurseryChunkUsableSize) - capacity();
1474
// Don't poison the trailer.
1475
MOZ_ASSERT(capacity() + poisonSize <= NurseryChunkUsableSize);
1476
chunk(0).poisonRange(capacity(), poisonSize, JS_FRESH_NURSERY_PATTERN,
1477
MemCheckKind::MakeUndefined);
1478
}
1479
1480
capacity_ = newCapacity;
1481
1482
setCurrentEnd();
1483
}
1484
1485
void js::Nursery::freeChunksFrom(const unsigned firstFreeChunk) {
1486
MOZ_ASSERT(firstFreeChunk < chunks_.length());
1487
1488
// The loop below may need to skip the first chunk, so we may use this so we
1489
// can modify it.
1490
unsigned firstChunkToDecommit = firstFreeChunk;
1491
1492
if ((firstChunkToDecommit == 0) && isSubChunkMode()) {
1493
// Part of the first chunk may be hard-decommitted, un-decommit it so that
1494
// the GC's normal chunk-handling doesn't segfault.
1495
MOZ_ASSERT(currentChunk_ == 0);
1496
if (!chunk(0).markPagesInUseHard(ChunkSize - ArenaSize)) {
1497
// Free the chunk if we can't allocate its pages.
1498
UnmapPages(static_cast<void*>(&chunk(0)), ChunkSize);
1499
firstChunkToDecommit = 1;
1500
}
1501
}
1502
1503
{
1504
AutoLockHelperThreadState lock;
1505
for (size_t i = firstChunkToDecommit; i < chunks_.length(); i++) {
1506
decommitTask.queueChunk(chunks_[i], lock);
1507
}
1508
decommitTask.startOrRunIfIdle(lock);
1509
}
1510
1511
chunks_.shrinkTo(firstFreeChunk);
1512
}
1513
1514
void js::Nursery::shrinkAllocableSpace(size_t newCapacity) {
1515
#ifdef JS_GC_ZEAL
1516
if (runtime()->hasZealMode(ZealMode::GenerationalGC)) {
1517
return;
1518
}
1519
#endif
1520
1521
// Don't shrink the nursery to zero (use Nursery::disable() instead)
1522
// This can't happen due to the rounding-down performed above because of the
1523
// clamping in maybeResizeNursery().
1524
MOZ_ASSERT(newCapacity != 0);
1525
// Don't attempt to shrink it to the same size.
1526
if (newCapacity == capacity_) {
1527
return;
1528
}
1529
MOZ_ASSERT(newCapacity < capacity_);
1530
1531
unsigned newCount = JS_HOWMANY(newCapacity, ChunkSize);
1532
if (newCount < allocatedChunkCount()) {
1533
freeChunksFrom(newCount);
1534
}
1535
1536
size_t oldCapacity = capacity_;
1537
capacity_ = newCapacity;
1538
1539
setCurrentEnd();
1540
1541
if (isSubChunkMode()) {
1542
MOZ_ASSERT(currentChunk_ == 0);
1543
chunk(0).poisonRange(newCapacity,
1544
Min(oldCapacity, NurseryChunkUsableSize) - newCapacity,
1545
JS_SWEPT_NURSERY_PATTERN, MemCheckKind::MakeNoAccess);
1546
1547
AutoLockHelperThreadState lock;
1548
decommitTask.queueRange(capacity_, chunk(0), lock);
1549
decommitTask.startOrRunIfIdle(lock);
1550
}
1551
}
1552
1553
void js::Nursery::minimizeAllocableSpace() {
1554
shrinkAllocableSpace(roundSize(tunables().gcMinNurseryBytes()));
1555
}
1556
1557
bool js::Nursery::queueDictionaryModeObjectToSweep(NativeObject* obj) {
1558
MOZ_ASSERT(IsInsideNursery(obj));
1559
return dictionaryModeObjects_.append(obj);
1560
}
1561
1562
uintptr_t js::Nursery::currentEnd() const {
1563
// These are separate asserts because it can be useful to see which one
1564
// failed.
1565
MOZ_ASSERT_IF(isSubChunkMode(), currentChunk_ == 0);
1566
MOZ_ASSERT_IF(isSubChunkMode(), currentEnd_ <= chunk(currentChunk_).end());
1567
MOZ_ASSERT_IF(!isSubChunkMode(), currentEnd_ == chunk(currentChunk_).end());
1568
MOZ_ASSERT(currentEnd_ != chunk(currentChunk_).start());
1569
return currentEnd_;
1570
}
1571
1572
gcstats::Statistics& js::Nursery::stats() const {
1573
return runtime()->gc.stats();
1574
}
1575
1576
MOZ_ALWAYS_INLINE const js::gc::GCSchedulingTunables& js::Nursery::tunables()
1577
const {
1578
return runtime()->gc.tunables;
1579
}
1580
1581
bool js::Nursery::isSubChunkMode() const {
1582
return capacity() <= NurseryChunkUsableSize;
1583
}
1584
1585
void js::Nursery::sweepDictionaryModeObjects() {
1586
for (auto obj : dictionaryModeObjects_) {
1587
if (!IsForwarded(obj)) {
1588
obj->sweepDictionaryListPointer();
1589
} else {
1590
Forwarded(obj)->updateDictionaryListPointerAfterMinorGC(obj);
1591
}
1592
}
1593
dictionaryModeObjects_.clear();
1594
}
1595
1596
void js::Nursery::sweepMapAndSetObjects() {
1597
auto fop = runtime_->defaultFreeOp();
1598
1599
for (auto mapobj : mapsWithNurseryMemory_) {
1600
MapObject::sweepAfterMinorGC(fop, mapobj);
1601
}
1602
mapsWithNurseryMemory_.clearAndFree();
1603
1604
for (auto setobj : setsWithNurseryMemory_) {
1605
SetObject::sweepAfterMinorGC(fop, setobj);
1606
}
1607
setsWithNurseryMemory_.clearAndFree();
1608
}
1609
1610
JS_PUBLIC_API void JS::EnableNurseryStrings(JSContext* cx) {
1611
AutoEmptyNursery empty(cx);
1612
ReleaseAllJITCode(cx->runtime()->defaultFreeOp());
1613
cx->runtime()->gc.nursery().enableStrings();
1614
}
1615
1616
JS_PUBLIC_API void JS::DisableNurseryStrings(JSContext* cx) {
1617
AutoEmptyNursery empty(cx);
1618
ReleaseAllJITCode(cx->runtime()->defaultFreeOp());
1619
cx->runtime()->gc.nursery().disableStrings();
1620
}