Source code

Revision control

Other Tools

1
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2
* vim: set ts=8 sts=2 et sw=2 tw=80:
3
*
4
* Copyright 2015 Mozilla Foundation
5
*
6
* Licensed under the Apache License, Version 2.0 (the "License");
7
* you may not use this file except in compliance with the License.
8
* You may obtain a copy of the License at
9
*
11
*
12
* Unless required by applicable law or agreed to in writing, software
13
* distributed under the License is distributed on an "AS IS" BASIS,
14
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
* See the License for the specific language governing permissions and
16
* limitations under the License.
17
*/
18
19
#include "wasm/WasmGenerator.h"
20
21
#include "mozilla/CheckedInt.h"
22
#include "mozilla/EnumeratedRange.h"
23
#include "mozilla/SHA1.h"
24
#include "mozilla/Unused.h"
25
26
#include <algorithm>
27
#include <thread>
28
29
#include "util/Memory.h"
30
#include "util/Text.h"
31
#include "wasm/WasmBaselineCompile.h"
32
#include "wasm/WasmCompile.h"
33
#include "wasm/WasmCraneliftCompile.h"
34
#include "wasm/WasmIonCompile.h"
35
#include "wasm/WasmStubs.h"
36
37
#include "jit/MacroAssembler-inl.h"
38
39
using namespace js;
40
using namespace js::jit;
41
using namespace js::wasm;
42
43
using mozilla::CheckedInt;
44
using mozilla::MakeEnumeratedRange;
45
using mozilla::Unused;
46
47
bool CompiledCode::swap(MacroAssembler& masm) {
48
MOZ_ASSERT(bytes.empty());
49
if (!masm.swapBuffer(bytes)) {
50
return false;
51
}
52
53
callSites.swap(masm.callSites());
54
callSiteTargets.swap(masm.callSiteTargets());
55
trapSites.swap(masm.trapSites());
56
symbolicAccesses.swap(masm.symbolicAccesses());
57
codeLabels.swap(masm.codeLabels());
58
return true;
59
}
60
61
// ****************************************************************************
62
// ModuleGenerator
63
64
static const unsigned GENERATOR_LIFO_DEFAULT_CHUNK_SIZE = 4 * 1024;
65
static const unsigned COMPILATION_LIFO_DEFAULT_CHUNK_SIZE = 64 * 1024;
66
static const uint32_t BAD_CODE_RANGE = UINT32_MAX;
67
68
ModuleGenerator::ModuleGenerator(const CompileArgs& args,
69
ModuleEnvironment* env,
70
const Atomic<bool>* cancelled,
71
UniqueChars* error)
72
: compileArgs_(&args),
73
error_(error),
74
cancelled_(cancelled),
75
env_(env),
76
linkData_(nullptr),
77
metadataTier_(nullptr),
78
taskState_(mutexid::WasmCompileTaskState),
79
lifo_(GENERATOR_LIFO_DEFAULT_CHUNK_SIZE),
80
masmAlloc_(&lifo_),
81
masm_(masmAlloc_, /* limitedSize= */ false),
82
debugTrapCodeOffset_(),
83
lastPatchedCallSite_(0),
84
startOfUnpatchedCallsites_(0),
85
parallel_(false),
86
outstanding_(0),
87
currentTask_(nullptr),
88
batchedBytecode_(0),
89
finishedFuncDefs_(false) {
90
MOZ_ASSERT(IsCompilingWasm());
91
}
92
93
ModuleGenerator::~ModuleGenerator() {
94
MOZ_ASSERT_IF(finishedFuncDefs_, !batchedBytecode_);
95
MOZ_ASSERT_IF(finishedFuncDefs_, !currentTask_);
96
97
if (parallel_) {
98
if (outstanding_) {
99
// Remove any pending compilation tasks from the worklist.
100
{
101
AutoLockHelperThreadState lock;
102
CompileTaskPtrFifo& worklist =
103
HelperThreadState().wasmWorklist(lock, mode());
104
auto pred = [this](CompileTask* task) {
105
return &task->state == &taskState_;
106
};
107
size_t removed = worklist.eraseIf(pred);
108
MOZ_ASSERT(outstanding_ >= removed);
109
outstanding_ -= removed;
110
}
111
112
// Wait until all active compilation tasks have finished.
113
{
114
auto taskState = taskState_.lock();
115
while (true) {
116
MOZ_ASSERT(outstanding_ >= taskState->finished.length());
117
outstanding_ -= taskState->finished.length();
118
taskState->finished.clear();
119
120
MOZ_ASSERT(outstanding_ >= taskState->numFailed);
121
outstanding_ -= taskState->numFailed;
122
taskState->numFailed = 0;
123
124
if (!outstanding_) {
125
break;
126
}
127
128
taskState.wait(/* failed or finished */);
129
}
130
}
131
}
132
} else {
133
MOZ_ASSERT(!outstanding_);
134
}
135
136
// Propagate error state.
137
if (error_ && !*error_) {
138
*error_ = std::move(taskState_.lock()->errorMessage);
139
}
140
}
141
142
bool ModuleGenerator::allocateGlobalBytes(uint32_t bytes, uint32_t align,
143
uint32_t* globalDataOffset) {
144
CheckedInt<uint32_t> newGlobalDataLength(metadata_->globalDataLength);
145
146
newGlobalDataLength +=
147
ComputeByteAlignment(newGlobalDataLength.value(), align);
148
if (!newGlobalDataLength.isValid()) {
149
return false;
150
}
151
152
*globalDataOffset = newGlobalDataLength.value();
153
newGlobalDataLength += bytes;
154
155
if (!newGlobalDataLength.isValid()) {
156
return false;
157
}
158
159
metadata_->globalDataLength = newGlobalDataLength.value();
160
return true;
161
}
162
163
bool ModuleGenerator::init(Metadata* maybeAsmJSMetadata) {
164
// Perform fallible metadata, linkdata, assumption allocations.
165
166
MOZ_ASSERT(isAsmJS() == !!maybeAsmJSMetadata);
167
if (maybeAsmJSMetadata) {
168
metadata_ = maybeAsmJSMetadata;
169
} else {
170
metadata_ = js_new<Metadata>();
171
if (!metadata_) {
172
return false;
173
}
174
}
175
176
if (compileArgs_->scriptedCaller.filename) {
177
metadata_->filename =
178
DuplicateString(compileArgs_->scriptedCaller.filename.get());
179
if (!metadata_->filename) {
180
return false;
181
}
182
183
metadata_->filenameIsURL = compileArgs_->scriptedCaller.filenameIsURL;
184
} else {
185
MOZ_ASSERT(!compileArgs_->scriptedCaller.filenameIsURL);
186
}
187
188
if (compileArgs_->sourceMapURL) {
189
metadata_->sourceMapURL = DuplicateString(compileArgs_->sourceMapURL.get());
190
if (!metadata_->sourceMapURL) {
191
return false;
192
}
193
}
194
195
linkData_ = js::MakeUnique<LinkData>(tier());
196
if (!linkData_) {
197
return false;
198
}
199
200
metadataTier_ = js::MakeUnique<MetadataTier>(tier());
201
if (!metadataTier_) {
202
return false;
203
}
204
205
// funcToCodeRange maps function indices to code-range indices and all
206
// elements will be initialized by the time module generation is finished.
207
208
if (!metadataTier_->funcToCodeRange.appendN(BAD_CODE_RANGE,
209
env_->funcTypes.length())) {
210
return false;
211
}
212
213
// Pre-reserve space for large Vectors to avoid the significant cost of the
214
// final reallocs. In particular, the MacroAssembler can be enormous, so be
215
// extra conservative. Since large over-reservations may fail when the
216
// actual allocations will succeed, ignore OOM failures. Note,
217
// podResizeToFit calls at the end will trim off unneeded capacity.
218
219
size_t codeSectionSize = env_->codeSection ? env_->codeSection->size : 0;
220
221
size_t estimatedCodeSize =
222
1.2 * EstimateCompiledCodeSize(tier(), codeSectionSize);
223
Unused << masm_.reserve(std::min(estimatedCodeSize, MaxCodeBytesPerProcess));
224
225
Unused << metadataTier_->codeRanges.reserve(2 * env_->numFuncDefs());
226
227
const size_t ByteCodesPerCallSite = 50;
228
Unused << metadataTier_->callSites.reserve(codeSectionSize /
229
ByteCodesPerCallSite);
230
231
const size_t ByteCodesPerOOBTrap = 10;
232
Unused << metadataTier_->trapSites[Trap::OutOfBounds].reserve(
233
codeSectionSize / ByteCodesPerOOBTrap);
234
235
// Allocate space in TlsData for declarations that need it.
236
237
MOZ_ASSERT(metadata_->globalDataLength == 0);
238
239
for (size_t i = 0; i < env_->funcImportGlobalDataOffsets.length(); i++) {
240
uint32_t globalDataOffset;
241
if (!allocateGlobalBytes(sizeof(FuncImportTls), sizeof(void*),
242
&globalDataOffset)) {
243
return false;
244
}
245
246
env_->funcImportGlobalDataOffsets[i] = globalDataOffset;
247
248
FuncType copy;
249
if (!copy.clone(*env_->funcTypes[i])) {
250
return false;
251
}
252
if (!metadataTier_->funcImports.emplaceBack(std::move(copy),
253
globalDataOffset)) {
254
return false;
255
}
256
}
257
258
for (TableDesc& table : env_->tables) {
259
if (!allocateGlobalBytes(sizeof(TableTls), sizeof(void*),
260
&table.globalDataOffset)) {
261
return false;
262
}
263
}
264
265
if (!isAsmJS()) {
266
for (TypeDef& td : env_->types) {
267
if (!td.isFuncType()) {
268
continue;
269
}
270
271
FuncTypeWithId& funcType = td.funcType();
272
if (FuncTypeIdDesc::isGlobal(funcType)) {
273
uint32_t globalDataOffset;
274
if (!allocateGlobalBytes(sizeof(void*), sizeof(void*),
275
&globalDataOffset)) {
276
return false;
277
}
278
279
funcType.id = FuncTypeIdDesc::global(funcType, globalDataOffset);
280
281
FuncType copy;
282
if (!copy.clone(funcType)) {
283
return false;
284
}
285
286
if (!metadata_->funcTypeIds.emplaceBack(std::move(copy), funcType.id)) {
287
return false;
288
}
289
} else {
290
funcType.id = FuncTypeIdDesc::immediate(funcType);
291
}
292
}
293
}
294
295
for (GlobalDesc& global : env_->globals) {
296
if (global.isConstant()) {
297
continue;
298
}
299
300
uint32_t width =
301
global.isIndirect() ? sizeof(void*) : SizeOf(global.type());
302
303
uint32_t globalDataOffset;
304
if (!allocateGlobalBytes(width, width, &globalDataOffset)) {
305
return false;
306
}
307
308
global.setOffset(globalDataOffset);
309
}
310
311
// Accumulate all exported functions:
312
// - explicitly marked as such;
313
// - implicitly exported by being an element of function tables;
314
// - implicitly exported by being the start function;
315
// The FuncExportVector stored in Metadata needs to be sorted (to allow
316
// O(log(n)) lookup at runtime) and deduplicated. Use a vector with invalid
317
// entries for every single function, that we'll fill as we go through the
318
// exports, and in which we'll remove invalid entries after the fact.
319
320
static_assert(((uint64_t(MaxFuncs) << 1) | 1) < uint64_t(UINT32_MAX),
321
"bit packing won't work in ExportedFunc");
322
323
class ExportedFunc {
324
uint32_t value;
325
326
public:
327
ExportedFunc() : value(UINT32_MAX) {}
328
ExportedFunc(uint32_t index, bool isExplicit)
329
: value((index << 1) | (isExplicit ? 1 : 0)) {}
330
uint32_t index() const { return value >> 1; }
331
bool isExplicit() const { return value & 0x1; }
332
bool operator<(const ExportedFunc& other) const {
333
return index() < other.index();
334
}
335
bool operator==(const ExportedFunc& other) const {
336
return index() == other.index();
337
}
338
bool isInvalid() const { return value == UINT32_MAX; }
339
void mergeExplicit(bool explicitBit) {
340
if (!isExplicit() && explicitBit) {
341
value |= 0x1;
342
}
343
}
344
};
345
346
Vector<ExportedFunc, 8, SystemAllocPolicy> exportedFuncs;
347
if (!exportedFuncs.resize(env_->numFuncs())) {
348
return false;
349
}
350
351
auto addOrMerge = [&exportedFuncs](ExportedFunc newEntry) {
352
uint32_t index = newEntry.index();
353
if (exportedFuncs[index].isInvalid()) {
354
exportedFuncs[index] = newEntry;
355
} else {
356
exportedFuncs[index].mergeExplicit(newEntry.isExplicit());
357
}
358
};
359
360
for (const Export& exp : env_->exports) {
361
if (exp.kind() == DefinitionKind::Function) {
362
addOrMerge(ExportedFunc(exp.funcIndex(), true));
363
}
364
}
365
366
if (env_->startFuncIndex) {
367
addOrMerge(ExportedFunc(*env_->startFuncIndex, true));
368
}
369
370
for (const ElemSegment* seg : env_->elemSegments) {
371
// For now, the segments always carry function indices regardless of the
372
// segment's declared element type; this works because the only legal
373
// element types are funcref and anyref and the only legal values are
374
// functions and null. We always add functions in segments as exported
375
// functions, regardless of the segment's type. In the future, if we make
376
// the representation of AnyRef segments different, we will have to consider
377
// function values in those segments specially.
378
bool isAsmJS =
379
seg->active() && env_->tables[seg->tableIndex].kind == TableKind::AsmJS;
380
if (!isAsmJS) {
381
for (uint32_t funcIndex : seg->elemFuncIndices) {
382
if (funcIndex == NullFuncIndex) {
383
continue;
384
}
385
addOrMerge(ExportedFunc(funcIndex, false));
386
}
387
}
388
}
389
390
auto* newEnd =
391
std::remove_if(exportedFuncs.begin(), exportedFuncs.end(),
392
[](const ExportedFunc& exp) { return exp.isInvalid(); });
393
exportedFuncs.erase(newEnd, exportedFuncs.end());
394
395
if (!metadataTier_->funcExports.reserve(exportedFuncs.length())) {
396
return false;
397
}
398
399
for (const ExportedFunc& funcIndex : exportedFuncs) {
400
FuncType funcType;
401
if (!funcType.clone(*env_->funcTypes[funcIndex.index()])) {
402
return false;
403
}
404
metadataTier_->funcExports.infallibleEmplaceBack(
405
std::move(funcType), funcIndex.index(), funcIndex.isExplicit());
406
}
407
408
// Determine whether parallel or sequential compilation is to be used and
409
// initialize the CompileTasks that will be used in either mode.
410
411
GlobalHelperThreadState& threads = HelperThreadState();
412
MOZ_ASSERT(threads.threadCount > 1);
413
414
uint32_t numTasks;
415
if (CanUseExtraThreads() && threads.cpuCount > 1) {
416
parallel_ = true;
417
numTasks = 2 * threads.maxWasmCompilationThreads();
418
} else {
419
numTasks = 1;
420
}
421
422
if (!tasks_.initCapacity(numTasks)) {
423
return false;
424
}
425
for (size_t i = 0; i < numTasks; i++) {
426
tasks_.infallibleEmplaceBack(*env_, taskState_,
427
COMPILATION_LIFO_DEFAULT_CHUNK_SIZE);
428
}
429
430
if (!freeTasks_.reserve(numTasks)) {
431
return false;
432
}
433
for (size_t i = 0; i < numTasks; i++) {
434
freeTasks_.infallibleAppend(&tasks_[i]);
435
}
436
437
// Fill in function stubs for each import so that imported functions can be
438
// used in all the places that normal function definitions can (table
439
// elements, export calls, etc).
440
441
CompiledCode& importCode = tasks_[0].output;
442
MOZ_ASSERT(importCode.empty());
443
444
if (!GenerateImportFunctions(*env_, metadataTier_->funcImports,
445
&importCode)) {
446
return false;
447
}
448
449
if (!linkCompiledCode(importCode)) {
450
return false;
451
}
452
453
importCode.clear();
454
return true;
455
}
456
457
bool ModuleGenerator::funcIsCompiled(uint32_t funcIndex) const {
458
return metadataTier_->funcToCodeRange[funcIndex] != BAD_CODE_RANGE;
459
}
460
461
const CodeRange& ModuleGenerator::funcCodeRange(uint32_t funcIndex) const {
462
MOZ_ASSERT(funcIsCompiled(funcIndex));
463
const CodeRange& cr =
464
metadataTier_->codeRanges[metadataTier_->funcToCodeRange[funcIndex]];
465
MOZ_ASSERT(cr.isFunction());
466
return cr;
467
}
468
469
static bool InRange(uint32_t caller, uint32_t callee) {
470
// We assume JumpImmediateRange is defined conservatively enough that the
471
// slight difference between 'caller' (which is really the return address
472
// offset) and the actual base of the relative displacement computation
473
// isn't significant.
474
uint32_t range = std::min(JitOptions.jumpThreshold, JumpImmediateRange);
475
if (caller < callee) {
476
return callee - caller < range;
477
}
478
return caller - callee < range;
479
}
480
481
typedef HashMap<uint32_t, uint32_t, DefaultHasher<uint32_t>, SystemAllocPolicy>
482
OffsetMap;
483
typedef EnumeratedArray<Trap, Trap::Limit, Maybe<uint32_t>>
484
TrapMaybeOffsetArray;
485
486
bool ModuleGenerator::linkCallSites() {
487
masm_.haltingAlign(CodeAlignment);
488
489
// Create far jumps for calls that have relative offsets that may otherwise
490
// go out of range. This method is called both between function bodies (at a
491
// frequency determined by the ISA's jump range) and once at the very end of
492
// a module's codegen after all possible calls/traps have been emitted.
493
494
OffsetMap existingCallFarJumps;
495
for (; lastPatchedCallSite_ < metadataTier_->callSites.length();
496
lastPatchedCallSite_++) {
497
const CallSite& callSite = metadataTier_->callSites[lastPatchedCallSite_];
498
const CallSiteTarget& target = callSiteTargets_[lastPatchedCallSite_];
499
uint32_t callerOffset = callSite.returnAddressOffset();
500
switch (callSite.kind()) {
501
case CallSiteDesc::Dynamic:
502
case CallSiteDesc::Symbolic:
503
break;
504
case CallSiteDesc::Func: {
505
if (funcIsCompiled(target.funcIndex())) {
506
uint32_t calleeOffset =
507
funcCodeRange(target.funcIndex()).funcNormalEntry();
508
if (InRange(callerOffset, calleeOffset)) {
509
masm_.patchCall(callerOffset, calleeOffset);
510
break;
511
}
512
}
513
514
OffsetMap::AddPtr p =
515
existingCallFarJumps.lookupForAdd(target.funcIndex());
516
if (!p) {
517
Offsets offsets;
518
offsets.begin = masm_.currentOffset();
519
if (!callFarJumps_.emplaceBack(target.funcIndex(),
520
masm_.farJumpWithPatch())) {
521
return false;
522
}
523
offsets.end = masm_.currentOffset();
524
if (masm_.oom()) {
525
return false;
526
}
527
if (!metadataTier_->codeRanges.emplaceBack(CodeRange::FarJumpIsland,
528
offsets)) {
529
return false;
530
}
531
if (!existingCallFarJumps.add(p, target.funcIndex(), offsets.begin)) {
532
return false;
533
}
534
}
535
536
masm_.patchCall(callerOffset, p->value());
537
break;
538
}
539
case CallSiteDesc::Breakpoint:
540
case CallSiteDesc::EnterFrame:
541
case CallSiteDesc::LeaveFrame: {
542
Uint32Vector& jumps = metadataTier_->debugTrapFarJumpOffsets;
543
if (jumps.empty() || !InRange(jumps.back(), callerOffset)) {
544
// See BaseCompiler::insertBreakablePoint for why we must
545
// reload the TLS register on this path.
546
Offsets offsets;
547
offsets.begin = masm_.currentOffset();
548
masm_.loadPtr(Address(FramePointer, offsetof(Frame, tls)),
549
WasmTlsReg);
550
CodeOffset jumpOffset = masm_.farJumpWithPatch();
551
offsets.end = masm_.currentOffset();
552
if (masm_.oom()) {
553
return false;
554
}
555
if (!metadataTier_->codeRanges.emplaceBack(CodeRange::FarJumpIsland,
556
offsets)) {
557
return false;
558
}
559
if (!debugTrapFarJumps_.emplaceBack(jumpOffset)) {
560
return false;
561
}
562
if (!jumps.emplaceBack(offsets.begin)) {
563
return false;
564
}
565
}
566
break;
567
}
568
}
569
}
570
571
masm_.flushBuffer();
572
return !masm_.oom();
573
}
574
575
void ModuleGenerator::noteCodeRange(uint32_t codeRangeIndex,
576
const CodeRange& codeRange) {
577
switch (codeRange.kind()) {
578
case CodeRange::Function:
579
MOZ_ASSERT(metadataTier_->funcToCodeRange[codeRange.funcIndex()] ==
580
BAD_CODE_RANGE);
581
metadataTier_->funcToCodeRange[codeRange.funcIndex()] = codeRangeIndex;
582
break;
583
case CodeRange::InterpEntry:
584
metadataTier_->lookupFuncExport(codeRange.funcIndex())
585
.initEagerInterpEntryOffset(codeRange.begin());
586
break;
587
case CodeRange::JitEntry:
588
// Nothing to do: jit entries are linked in the jump tables.
589
break;
590
case CodeRange::ImportJitExit:
591
metadataTier_->funcImports[codeRange.funcIndex()].initJitExitOffset(
592
codeRange.begin());
593
break;
594
case CodeRange::ImportInterpExit:
595
metadataTier_->funcImports[codeRange.funcIndex()].initInterpExitOffset(
596
codeRange.begin());
597
break;
598
case CodeRange::DebugTrap:
599
MOZ_ASSERT(!debugTrapCodeOffset_);
600
debugTrapCodeOffset_ = codeRange.begin();
601
break;
602
case CodeRange::TrapExit:
603
MOZ_ASSERT(!linkData_->trapOffset);
604
linkData_->trapOffset = codeRange.begin();
605
break;
606
case CodeRange::Throw:
607
// Jumped to by other stubs, so nothing to do.
608
break;
609
case CodeRange::FarJumpIsland:
610
case CodeRange::BuiltinThunk:
611
MOZ_CRASH("Unexpected CodeRange kind");
612
}
613
}
614
615
template <class Vec, class Op>
616
static bool AppendForEach(Vec* dstVec, const Vec& srcVec, Op op) {
617
if (!dstVec->growByUninitialized(srcVec.length())) {
618
return false;
619
}
620
621
typedef typename Vec::ElementType T;
622
623
const T* src = srcVec.begin();
624
625
T* dstBegin = dstVec->begin();
626
T* dstEnd = dstVec->end();
627
T* dstStart = dstEnd - srcVec.length();
628
629
for (T* dst = dstStart; dst != dstEnd; dst++, src++) {
630
new (dst) T(*src);
631
op(dst - dstBegin, dst);
632
}
633
634
return true;
635
}
636
637
bool ModuleGenerator::linkCompiledCode(CompiledCode& code) {
638
// Before merging in new code, if calls in a prior code range might go out of
639
// range, insert far jumps to extend the range.
640
641
if (!InRange(startOfUnpatchedCallsites_,
642
masm_.size() + code.bytes.length())) {
643
startOfUnpatchedCallsites_ = masm_.size();
644
if (!linkCallSites()) {
645
return false;
646
}
647
}
648
649
// All code offsets in 'code' must be incremented by their position in the
650
// overall module when the code was appended.
651
652
masm_.haltingAlign(CodeAlignment);
653
const size_t offsetInModule = masm_.size();
654
if (!masm_.appendRawCode(code.bytes.begin(), code.bytes.length())) {
655
return false;
656
}
657
658
auto codeRangeOp = [=](uint32_t codeRangeIndex, CodeRange* codeRange) {
659
codeRange->offsetBy(offsetInModule);
660
noteCodeRange(codeRangeIndex, *codeRange);
661
};
662
if (!AppendForEach(&metadataTier_->codeRanges, code.codeRanges,
663
codeRangeOp)) {
664
return false;
665
}
666
667
auto callSiteOp = [=](uint32_t, CallSite* cs) {
668
cs->offsetBy(offsetInModule);
669
};
670
if (!AppendForEach(&metadataTier_->callSites, code.callSites, callSiteOp)) {
671
return false;
672
}
673
674
if (!callSiteTargets_.appendAll(code.callSiteTargets)) {
675
return false;
676
}
677
678
for (Trap trap : MakeEnumeratedRange(Trap::Limit)) {
679
auto trapSiteOp = [=](uint32_t, TrapSite* ts) {
680
ts->offsetBy(offsetInModule);
681
};
682
if (!AppendForEach(&metadataTier_->trapSites[trap], code.trapSites[trap],
683
trapSiteOp)) {
684
return false;
685
}
686
}
687
688
for (const SymbolicAccess& access : code.symbolicAccesses) {
689
uint32_t patchAt = offsetInModule + access.patchAt.offset();
690
if (!linkData_->symbolicLinks[access.target].append(patchAt)) {
691
return false;
692
}
693
}
694
695
for (const CodeLabel& codeLabel : code.codeLabels) {
696
LinkData::InternalLink link;
697
link.patchAtOffset = offsetInModule + codeLabel.patchAt().offset();
698
link.targetOffset = offsetInModule + codeLabel.target().offset();
699
#ifdef JS_CODELABEL_LINKMODE
700
link.mode = codeLabel.linkMode();
701
#endif
702
if (!linkData_->internalLinks.append(link)) {
703
return false;
704
}
705
}
706
707
for (size_t i = 0; i < code.stackMaps.length(); i++) {
708
StackMaps::Maplet maplet = code.stackMaps.move(i);
709
maplet.offsetBy(offsetInModule);
710
if (!metadataTier_->stackMaps.add(maplet)) {
711
// This function is now the only owner of maplet.map, so we'd better
712
// free it right now.
713
maplet.map->destroy();
714
return false;
715
}
716
}
717
718
return true;
719
}
720
721
static bool ExecuteCompileTask(CompileTask* task, UniqueChars* error) {
722
MOZ_ASSERT(task->lifo.isEmpty());
723
MOZ_ASSERT(task->output.empty());
724
725
switch (task->env.tier()) {
726
case Tier::Optimized:
727
#ifdef ENABLE_WASM_CRANELIFT
728
if (task->env.optimizedBackend() == OptimizedBackend::Cranelift) {
729
if (!CraneliftCompileFunctions(task->env, task->lifo, task->inputs,
730
&task->output, error)) {
731
return false;
732
}
733
break;
734
}
735
#endif
736
MOZ_ASSERT(task->env.optimizedBackend() == OptimizedBackend::Ion);
737
if (!IonCompileFunctions(task->env, task->lifo, task->inputs,
738
&task->output, error)) {
739
return false;
740
}
741
break;
742
case Tier::Baseline:
743
if (!BaselineCompileFunctions(task->env, task->lifo, task->inputs,
744
&task->output, error)) {
745
return false;
746
}
747
break;
748
}
749
750
MOZ_ASSERT(task->lifo.isEmpty());
751
MOZ_ASSERT(task->inputs.length() == task->output.codeRanges.length());
752
task->inputs.clear();
753
return true;
754
}
755
756
void wasm::ExecuteCompileTaskFromHelperThread(CompileTask* task) {
757
TraceLoggerThread* logger = TraceLoggerForCurrentThread();
758
AutoTraceLog logCompile(logger, TraceLogger_WasmCompilation);
759
760
UniqueChars error;
761
bool ok = ExecuteCompileTask(task, &error);
762
763
auto taskState = task->state.lock();
764
765
if (!ok || !taskState->finished.append(task)) {
766
taskState->numFailed++;
767
if (!taskState->errorMessage) {
768
taskState->errorMessage = std::move(error);
769
}
770
}
771
772
taskState.notify_one(/* failed or finished */);
773
}
774
775
bool ModuleGenerator::locallyCompileCurrentTask() {
776
if (!ExecuteCompileTask(currentTask_, error_)) {
777
return false;
778
}
779
if (!finishTask(currentTask_)) {
780
return false;
781
}
782
currentTask_ = nullptr;
783
batchedBytecode_ = 0;
784
return true;
785
}
786
787
bool ModuleGenerator::finishTask(CompileTask* task) {
788
masm_.haltingAlign(CodeAlignment);
789
790
if (!linkCompiledCode(task->output)) {
791
return false;
792
}
793
794
task->output.clear();
795
796
MOZ_ASSERT(task->inputs.empty());
797
MOZ_ASSERT(task->output.empty());
798
MOZ_ASSERT(task->lifo.isEmpty());
799
freeTasks_.infallibleAppend(task);
800
return true;
801
}
802
803
bool ModuleGenerator::launchBatchCompile() {
804
MOZ_ASSERT(currentTask_);
805
806
if (cancelled_ && *cancelled_) {
807
return false;
808
}
809
810
if (!parallel_) {
811
return locallyCompileCurrentTask();
812
}
813
814
if (!StartOffThreadWasmCompile(currentTask_, mode())) {
815
return false;
816
}
817
outstanding_++;
818
currentTask_ = nullptr;
819
batchedBytecode_ = 0;
820
return true;
821
}
822
823
bool ModuleGenerator::finishOutstandingTask() {
824
MOZ_ASSERT(parallel_);
825
826
CompileTask* task = nullptr;
827
{
828
auto taskState = taskState_.lock();
829
while (true) {
830
MOZ_ASSERT(outstanding_ > 0);
831
832
if (taskState->numFailed > 0) {
833
return false;
834
}
835
836
if (!taskState->finished.empty()) {
837
outstanding_--;
838
task = taskState->finished.popCopy();
839
break;
840
}
841
842
taskState.wait(/* failed or finished */);
843
}
844
}
845
846
// Call outside of the compilation lock.
847
return finishTask(task);
848
}
849
850
bool ModuleGenerator::compileFuncDef(uint32_t funcIndex,
851
uint32_t lineOrBytecode,
852
const uint8_t* begin, const uint8_t* end,
853
Uint32Vector&& lineNums) {
854
MOZ_ASSERT(!finishedFuncDefs_);
855
MOZ_ASSERT(funcIndex < env_->numFuncs());
856
857
uint32_t threshold;
858
switch (tier()) {
859
case Tier::Baseline:
860
threshold = JitOptions.wasmBatchBaselineThreshold;
861
break;
862
case Tier::Optimized:
863
switch (env_->optimizedBackend()) {
864
case OptimizedBackend::Ion:
865
threshold = JitOptions.wasmBatchIonThreshold;
866
break;
867
case OptimizedBackend::Cranelift:
868
threshold = JitOptions.wasmBatchCraneliftThreshold;
869
break;
870
default:
871
MOZ_CRASH("Invalid optimizedBackend value");
872
}
873
break;
874
default:
875
MOZ_CRASH("Invalid tier value");
876
break;
877
}
878
879
uint32_t funcBytecodeLength = end - begin;
880
881
// Do not go over the threshold if we can avoid it: spin off the compilation
882
// before appending the function if we would go over. (Very large single
883
// functions may still exceed the threshold but this is fine; it'll be very
884
// uncommon and is in any case safely handled by the MacroAssembler's buffer
885
// limit logic.)
886
887
if (currentTask_ && currentTask_->inputs.length() &&
888
batchedBytecode_ + funcBytecodeLength > threshold) {
889
if (!launchBatchCompile()) {
890
return false;
891
}
892
}
893
894
if (!currentTask_) {
895
if (freeTasks_.empty() && !finishOutstandingTask()) {
896
return false;
897
}
898
currentTask_ = freeTasks_.popCopy();
899
}
900
901
if (!currentTask_->inputs.emplaceBack(funcIndex, lineOrBytecode, begin, end,
902
std::move(lineNums))) {
903
return false;
904
}
905
906
batchedBytecode_ += funcBytecodeLength;
907
MOZ_ASSERT(batchedBytecode_ <= MaxCodeSectionBytes);
908
return true;
909
}
910
911
bool ModuleGenerator::finishFuncDefs() {
912
MOZ_ASSERT(!finishedFuncDefs_);
913
914
if (currentTask_ && !locallyCompileCurrentTask()) {
915
return false;
916
}
917
918
finishedFuncDefs_ = true;
919
return true;
920
}
921
922
bool ModuleGenerator::finishCodegen() {
923
// Now that all functions and stubs are generated and their CodeRanges
924
// known, patch all calls (which can emit far jumps) and far jumps. Linking
925
// can emit tiny far-jump stubs, so there is an ordering dependency here.
926
927
if (!linkCallSites()) {
928
return false;
929
}
930
931
for (CallFarJump far : callFarJumps_) {
932
masm_.patchFarJump(far.jump,
933
funcCodeRange(far.funcIndex).funcNormalEntry());
934
}
935
936
for (CodeOffset farJump : debugTrapFarJumps_) {
937
masm_.patchFarJump(farJump, debugTrapCodeOffset_);
938
}
939
940
// None of the linking or far-jump operations should emit masm metadata.
941
942
MOZ_ASSERT(masm_.callSites().empty());
943
MOZ_ASSERT(masm_.callSiteTargets().empty());
944
MOZ_ASSERT(masm_.trapSites().empty());
945
MOZ_ASSERT(masm_.symbolicAccesses().empty());
946
MOZ_ASSERT(masm_.codeLabels().empty());
947
948
masm_.finish();
949
return !masm_.oom();
950
}
951
952
bool ModuleGenerator::finishMetadataTier() {
953
// The stack maps aren't yet sorted. Do so now, since we'll need to
954
// binary-search them at GC time.
955
metadataTier_->stackMaps.sort();
956
957
#ifdef DEBUG
958
// Check that the stack map contains no duplicates, since that could lead to
959
// ambiguities about stack slot pointerness.
960
uint8_t* previousNextInsnAddr = nullptr;
961
for (size_t i = 0; i < metadataTier_->stackMaps.length(); i++) {
962
const StackMaps::Maplet& maplet = metadataTier_->stackMaps.get(i);
963
MOZ_ASSERT_IF(i > 0, uintptr_t(maplet.nextInsnAddr) >
964
uintptr_t(previousNextInsnAddr));
965
previousNextInsnAddr = maplet.nextInsnAddr;
966
}
967
968
// Assert all sorted metadata is sorted.
969
uint32_t last = 0;
970
for (const CodeRange& codeRange : metadataTier_->codeRanges) {
971
MOZ_ASSERT(codeRange.begin() >= last);
972
last = codeRange.end();
973
}
974
975
last = 0;
976
for (const CallSite& callSite : metadataTier_->callSites) {
977
MOZ_ASSERT(callSite.returnAddressOffset() >= last);
978
last = callSite.returnAddressOffset();
979
}
980
981
for (Trap trap : MakeEnumeratedRange(Trap::Limit)) {
982
last = 0;
983
for (const TrapSite& trapSite : metadataTier_->trapSites[trap]) {
984
MOZ_ASSERT(trapSite.pcOffset >= last);
985
last = trapSite.pcOffset;
986
}
987
}
988
989
last = 0;
990
for (uint32_t debugTrapFarJumpOffset :
991
metadataTier_->debugTrapFarJumpOffsets) {
992
MOZ_ASSERT(debugTrapFarJumpOffset >= last);
993
last = debugTrapFarJumpOffset;
994
}
995
#endif
996
997
// These Vectors can get large and the excess capacity can be significant,
998
// so realloc them down to size.
999
1000
metadataTier_->funcToCodeRange.podResizeToFit();
1001
metadataTier_->codeRanges.podResizeToFit();
1002
metadataTier_->callSites.podResizeToFit();
1003
metadataTier_->trapSites.podResizeToFit();
1004
metadataTier_->debugTrapFarJumpOffsets.podResizeToFit();
1005
for (Trap trap : MakeEnumeratedRange(Trap::Limit)) {
1006
metadataTier_->trapSites[trap].podResizeToFit();
1007
}
1008
1009
return true;
1010
}
1011
1012
UniqueCodeTier ModuleGenerator::finishCodeTier() {
1013
MOZ_ASSERT(finishedFuncDefs_);
1014
1015
while (outstanding_ > 0) {
1016
if (!finishOutstandingTask()) {
1017
return nullptr;
1018
}
1019
}
1020
1021
#ifdef DEBUG
1022
for (uint32_t codeRangeIndex : metadataTier_->funcToCodeRange) {
1023
MOZ_ASSERT(codeRangeIndex != BAD_CODE_RANGE);
1024
}
1025
#endif
1026
1027
// Now that all imports/exports are known, we can generate a special
1028
// CompiledCode containing stubs.
1029
1030
CompiledCode& stubCode = tasks_[0].output;
1031
MOZ_ASSERT(stubCode.empty());
1032
1033
if (!GenerateStubs(*env_, metadataTier_->funcImports,
1034
metadataTier_->funcExports, &stubCode)) {
1035
return nullptr;
1036
}
1037
1038
if (!linkCompiledCode(stubCode)) {
1039
return nullptr;
1040
}
1041
1042
// Finish linking and metadata.
1043
1044
if (!finishCodegen()) {
1045
return nullptr;
1046
}
1047
1048
if (!finishMetadataTier()) {
1049
return nullptr;
1050
}
1051
1052
UniqueModuleSegment segment =
1053
ModuleSegment::create(tier(), masm_, *linkData_);
1054
if (!segment) {
1055
return nullptr;
1056
}
1057
1058
metadataTier_->stackMaps.offsetBy(uintptr_t(segment->base()));
1059
1060
#ifdef DEBUG
1061
// Check that each stack map is associated with a plausible instruction.
1062
for (size_t i = 0; i < metadataTier_->stackMaps.length(); i++) {
1063
MOZ_ASSERT(IsValidStackMapKey(env_->debugEnabled(),
1064
metadataTier_->stackMaps.get(i).nextInsnAddr),
1065
"wasm stack map does not reference a valid insn");
1066
}
1067
#endif
1068
1069
return js::MakeUnique<CodeTier>(std::move(metadataTier_), std::move(segment));
1070
}
1071
1072
SharedMetadata ModuleGenerator::finishMetadata(const Bytes& bytecode) {
1073
// Finish initialization of Metadata, which is only needed for constructing
1074
// the initial Module, not for tier-2 compilation.
1075
MOZ_ASSERT(mode() != CompileMode::Tier2);
1076
1077
// Copy over data from the ModuleEnvironment.
1078
1079
metadata_->memoryUsage = env_->memoryUsage;
1080
metadata_->minMemoryLength = env_->minMemoryLength;
1081
metadata_->maxMemoryLength = env_->maxMemoryLength;
1082
metadata_->startFuncIndex = env_->startFuncIndex;
1083
metadata_->tables = std::move(env_->tables);
1084
metadata_->globals = std::move(env_->globals);
1085
metadata_->nameCustomSectionIndex = env_->nameCustomSectionIndex;
1086
metadata_->moduleName = env_->moduleName;
1087
metadata_->funcNames = std::move(env_->funcNames);
1088
metadata_->omitsBoundsChecks = env_->hugeMemoryEnabled();
1089
metadata_->bigIntEnabled = env_->bigIntEnabled();
1090
1091
// Copy over additional debug information.
1092
1093
if (env_->debugEnabled()) {
1094
metadata_->debugEnabled = true;
1095
1096
const size_t numFuncTypes = env_->funcTypes.length();
1097
if (!metadata_->debugFuncArgTypes.resize(numFuncTypes)) {
1098
return nullptr;
1099
}
1100
if (!metadata_->debugFuncReturnTypes.resize(numFuncTypes)) {
1101
return nullptr;
1102
}
1103
for (size_t i = 0; i < numFuncTypes; i++) {
1104
if (!metadata_->debugFuncArgTypes[i].appendAll(
1105
env_->funcTypes[i]->args())) {
1106
return nullptr;
1107
}
1108
if (!metadata_->debugFuncReturnTypes[i].appendAll(
1109
env_->funcTypes[i]->results())) {
1110
return nullptr;
1111
}
1112
}
1113
1114
static_assert(sizeof(ModuleHash) <= sizeof(mozilla::SHA1Sum::Hash),
1115
"The ModuleHash size shall not exceed the SHA1 hash size.");
1116
mozilla::SHA1Sum::Hash hash;
1117
mozilla::SHA1Sum sha1Sum;
1118
sha1Sum.update(bytecode.begin(), bytecode.length());
1119
sha1Sum.finish(hash);
1120
memcpy(metadata_->debugHash, hash, sizeof(ModuleHash));
1121
}
1122
1123
MOZ_ASSERT_IF(env_->nameCustomSectionIndex, !!metadata_->namePayload);
1124
1125
// Metadata shouldn't be mutably modified after finishMetadata().
1126
SharedMetadata metadata = metadata_;
1127
metadata_ = nullptr;
1128
return metadata;
1129
}
1130
1131
SharedModule ModuleGenerator::finishModule(
1132
const ShareableBytes& bytecode,
1133
JS::OptimizedEncodingListener* maybeTier2Listener) {
1134
MOZ_ASSERT(mode() == CompileMode::Once || mode() == CompileMode::Tier1);
1135
1136
UniqueCodeTier codeTier = finishCodeTier();
1137
if (!codeTier) {
1138
return nullptr;
1139
}
1140
1141
JumpTables jumpTables;
1142
if (!jumpTables.init(mode(), codeTier->segment(),
1143
codeTier->metadata().codeRanges)) {
1144
return nullptr;
1145
}
1146
1147
// Copy over data from the Bytecode, which is going away at the end of
1148
// compilation.
1149
1150
DataSegmentVector dataSegments;
1151
if (!dataSegments.reserve(env_->dataSegments.length())) {
1152
return nullptr;
1153
}
1154
for (const DataSegmentEnv& srcSeg : env_->dataSegments) {
1155
MutableDataSegment dstSeg = js_new<DataSegment>(srcSeg);
1156
if (!dstSeg) {
1157
return nullptr;
1158
}
1159
if (!dstSeg->bytes.append(bytecode.begin() + srcSeg.bytecodeOffset,
1160
srcSeg.length)) {
1161
return nullptr;
1162
}
1163
dataSegments.infallibleAppend(std::move(dstSeg));
1164
}
1165
1166
CustomSectionVector customSections;
1167
if (!customSections.reserve(env_->customSections.length())) {
1168
return nullptr;
1169
}
1170
for (const CustomSectionEnv& srcSec : env_->customSections) {
1171
CustomSection sec;
1172
if (!sec.name.append(bytecode.begin() + srcSec.nameOffset,
1173
srcSec.nameLength)) {
1174
return nullptr;
1175
}
1176
MutableBytes payload = js_new<ShareableBytes>();
1177
if (!payload) {
1178
return nullptr;
1179
}
1180
if (!payload->append(bytecode.begin() + srcSec.payloadOffset,
1181
srcSec.payloadLength)) {
1182
return nullptr;
1183
}
1184
sec.payload = std::move(payload);
1185
customSections.infallibleAppend(std::move(sec));
1186
}
1187
1188
if (env_->nameCustomSectionIndex) {
1189
metadata_->namePayload =
1190
customSections[*env_->nameCustomSectionIndex].payload;
1191
}
1192
1193
SharedMetadata metadata = finishMetadata(bytecode.bytes);
1194
if (!metadata) {
1195
return nullptr;
1196
}
1197
1198
StructTypeVector structTypes;
1199
for (TypeDef& td : env_->types) {
1200
if (td.isStructType() && !structTypes.append(std::move(td.structType()))) {
1201
return nullptr;
1202
}
1203
}
1204
1205
MutableCode code =
1206
js_new<Code>(std::move(codeTier), *metadata, std::move(jumpTables),
1207
std::move(structTypes));
1208
if (!code || !code->initialize(*linkData_)) {
1209
return nullptr;
1210
}
1211
1212
// See Module debugCodeClaimed_ comments for why we need to make a separate
1213
// debug copy.
1214
1215
UniqueBytes debugUnlinkedCode;
1216
UniqueLinkData debugLinkData;
1217
const ShareableBytes* debugBytecode = nullptr;
1218
if (env_->debugEnabled()) {
1219
MOZ_ASSERT(mode() == CompileMode::Once);
1220
MOZ_ASSERT(tier() == Tier::Debug);
1221
1222
debugUnlinkedCode = js::MakeUnique<Bytes>();
1223
if (!debugUnlinkedCode || !debugUnlinkedCode->resize(masm_.bytesNeeded())) {
1224
return nullptr;
1225
}
1226
1227
masm_.executableCopy(debugUnlinkedCode->begin());
1228
1229
debugLinkData = std::move(linkData_);
1230
debugBytecode = &bytecode;
1231
}
1232
1233
// All the components are finished, so create the complete Module and start
1234
// tier-2 compilation if requested.
1235
1236
MutableModule module =
1237
js_new<Module>(*code, std::move(env_->imports), std::move(env_->exports),
1238
std::move(dataSegments), std::move(env_->elemSegments),
1239
std::move(customSections), std::move(debugUnlinkedCode),
1240
std::move(debugLinkData), debugBytecode);
1241
if (!module) {
1242
return nullptr;
1243
}
1244
1245
if (mode() == CompileMode::Tier1) {
1246
module->startTier2(*compileArgs_, bytecode, maybeTier2Listener);
1247
} else if (tier() == Tier::Serialized && maybeTier2Listener) {
1248
module->serialize(*linkData_, *maybeTier2Listener);
1249
}
1250
1251
return module;
1252
}
1253
1254
bool ModuleGenerator::finishTier2(const Module& module) {
1255
MOZ_ASSERT(mode() == CompileMode::Tier2);
1256
MOZ_ASSERT(tier() == Tier::Optimized);
1257
MOZ_ASSERT(!env_->debugEnabled());
1258
1259
if (cancelled_ && *cancelled_) {
1260
return false;
1261
}
1262
1263
UniqueCodeTier codeTier = finishCodeTier();
1264
if (!codeTier) {
1265
return false;
1266
}
1267
1268
if (MOZ_UNLIKELY(JitOptions.wasmDelayTier2)) {
1269
// Introduce an artificial delay when testing wasmDelayTier2, since we
1270
// want to exercise both tier1 and tier2 code in this case.
1271
std::this_thread::sleep_for(std::chrono::milliseconds(500));
1272
}
1273
1274
return module.finishTier2(*linkData_, std::move(codeTier));
1275
}
1276
1277
void CompileTask::runTask() { ExecuteCompileTaskFromHelperThread(this); }
1278
1279
size_t CompiledCode::sizeOfExcludingThis(
1280
mozilla::MallocSizeOf mallocSizeOf) const {
1281
size_t trapSitesSize = 0;
1282
for (const TrapSiteVector& vec : trapSites) {
1283
trapSitesSize += vec.sizeOfExcludingThis(mallocSizeOf);
1284
}
1285
1286
return bytes.sizeOfExcludingThis(mallocSizeOf) +
1287
codeRanges.sizeOfExcludingThis(mallocSizeOf) +
1288
callSites.sizeOfExcludingThis(mallocSizeOf) +
1289
callSiteTargets.sizeOfExcludingThis(mallocSizeOf) + trapSitesSize +
1290
symbolicAccesses.sizeOfExcludingThis(mallocSizeOf) +
1291
codeLabels.sizeOfExcludingThis(mallocSizeOf);
1292
}
1293
1294
size_t CompileTask::sizeOfExcludingThis(
1295
mozilla::MallocSizeOf mallocSizeOf) const {
1296
return lifo.sizeOfExcludingThis(mallocSizeOf) +
1297
inputs.sizeOfExcludingThis(mallocSizeOf) +
1298
output.sizeOfExcludingThis(mallocSizeOf);
1299
}