Source code

Revision control

Other Tools

1
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2
* vim: set ts=8 sts=2 et sw=2 tw=80:
3
*
4
* Copyright 2018 Mozilla Foundation
5
*
6
* Licensed under the Apache License, Version 2.0 (the "License");
7
* you may not use this file except in compliance with the License.
8
* You may obtain a copy of the License at
9
*
11
*
12
* Unless required by applicable law or agreed to in writing, software
13
* distributed under the License is distributed on an "AS IS" BASIS,
14
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
* See the License for the specific language governing permissions and
16
* limitations under the License.
17
*/
18
19
#include "wasm/WasmCraneliftCompile.h"
20
21
#include "mozilla/ScopeExit.h"
22
23
#include "jit/Disassemble.h"
24
#include "js/Printf.h"
25
26
#include "wasm/cranelift/baldrapi.h"
27
#include "wasm/cranelift/clifapi.h"
28
#include "wasm/WasmFrameIter.h" // js::wasm::GenerateFunction{Pro,Epi}logue
29
#include "wasm/WasmGenerator.h"
30
31
#include "jit/MacroAssembler-inl.h"
32
33
using namespace js;
34
using namespace js::jit;
35
using namespace js::wasm;
36
37
bool wasm::CraneliftCanCompile() {
38
#ifdef JS_CODEGEN_X64
39
return true;
40
#else
41
return false;
42
#endif
43
}
44
45
static inline SymbolicAddress ToSymbolicAddress(BD_SymbolicAddress bd) {
46
switch (bd) {
47
case BD_SymbolicAddress::MemoryGrow:
48
return SymbolicAddress::MemoryGrow;
49
case BD_SymbolicAddress::MemorySize:
50
return SymbolicAddress::MemorySize;
51
case BD_SymbolicAddress::MemoryCopy:
52
return SymbolicAddress::MemCopy;
53
case BD_SymbolicAddress::MemoryCopyShared:
54
return SymbolicAddress::MemCopyShared;
55
case BD_SymbolicAddress::DataDrop:
56
return SymbolicAddress::DataDrop;
57
case BD_SymbolicAddress::MemoryFill:
58
return SymbolicAddress::MemFill;
59
case BD_SymbolicAddress::MemoryFillShared:
60
return SymbolicAddress::MemFillShared;
61
case BD_SymbolicAddress::MemoryInit:
62
return SymbolicAddress::MemInit;
63
case BD_SymbolicAddress::TableCopy:
64
return SymbolicAddress::TableCopy;
65
case BD_SymbolicAddress::ElemDrop:
66
return SymbolicAddress::ElemDrop;
67
case BD_SymbolicAddress::TableFill:
68
return SymbolicAddress::TableFill;
69
case BD_SymbolicAddress::TableGet:
70
return SymbolicAddress::TableGet;
71
case BD_SymbolicAddress::TableGrow:
72
return SymbolicAddress::TableGrow;
73
case BD_SymbolicAddress::TableInit:
74
return SymbolicAddress::TableInit;
75
case BD_SymbolicAddress::TableSet:
76
return SymbolicAddress::TableSet;
77
case BD_SymbolicAddress::TableSize:
78
return SymbolicAddress::TableSize;
79
case BD_SymbolicAddress::FloorF32:
80
return SymbolicAddress::FloorF;
81
case BD_SymbolicAddress::FloorF64:
82
return SymbolicAddress::FloorD;
83
case BD_SymbolicAddress::CeilF32:
84
return SymbolicAddress::CeilF;
85
case BD_SymbolicAddress::CeilF64:
86
return SymbolicAddress::CeilD;
87
case BD_SymbolicAddress::NearestF32:
88
return SymbolicAddress::NearbyIntF;
89
case BD_SymbolicAddress::NearestF64:
90
return SymbolicAddress::NearbyIntD;
91
case BD_SymbolicAddress::TruncF32:
92
return SymbolicAddress::TruncF;
93
case BD_SymbolicAddress::TruncF64:
94
return SymbolicAddress::TruncD;
95
case BD_SymbolicAddress::Limit:
96
break;
97
}
98
MOZ_CRASH("unknown baldrdash symbolic address");
99
}
100
101
static bool GenerateCraneliftCode(WasmMacroAssembler& masm,
102
const CraneliftCompiledFunc& func,
103
const FuncTypeIdDesc& funcTypeId,
104
uint32_t lineOrBytecode,
105
uint32_t funcBytecodeSize,
106
FuncOffsets* offsets) {
107
wasm::GenerateFunctionPrologue(masm, funcTypeId, mozilla::Nothing(), offsets);
108
109
// Omit the check when framePushed is small and we know there's no
110
// recursion.
111
if (func.framePushed < MAX_UNCHECKED_LEAF_FRAME_SIZE && !func.containsCalls) {
112
masm.reserveStack(func.framePushed);
113
} else {
114
masm.wasmReserveStackChecked(func.framePushed,
115
BytecodeOffset(lineOrBytecode));
116
}
117
MOZ_ASSERT(masm.framePushed() == func.framePushed);
118
119
// Copy the machine code; handle jump tables and other read-only data below.
120
uint32_t funcBase = masm.currentOffset();
121
if (!masm.appendRawCode(func.code, func.codeSize)) {
122
return false;
123
}
124
#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
125
uint32_t codeEnd = masm.currentOffset();
126
#endif
127
128
wasm::GenerateFunctionEpilogue(masm, func.framePushed, offsets);
129
130
if (func.numRodataRelocs > 0) {
131
#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86)
132
constexpr size_t jumptableElementSize = 4;
133
134
MOZ_ASSERT(func.jumptablesSize % jumptableElementSize == 0);
135
136
// Align the jump tables properly.
137
masm.haltingAlign(jumptableElementSize);
138
139
// Copy over the tables and read-only data.
140
uint32_t rodataBase = masm.currentOffset();
141
if (!masm.appendRawCode(func.code + func.codeSize,
142
func.totalSize - func.codeSize)) {
143
return false;
144
}
145
146
uint32_t numElem = func.jumptablesSize / jumptableElementSize;
147
uint32_t bias = rodataBase - codeEnd;
148
149
// Bias the jump table(s). The table values are negative values
150
// representing backward jumps. By shifting the table down we increase the
151
// distance and so we add a negative value to reflect the larger distance.
152
//
153
// Note addToPCRel4() works from the end of the instruction, hence the loop
154
// bounds.
155
for (uint32_t i = 1; i <= numElem; i++) {
156
masm.addToPCRel4(rodataBase + (i * jumptableElementSize), -bias);
157
}
158
159
// Patch up the code locations. These represent forward distances that also
160
// become greater, so we add a positive value.
161
for (uint32_t i = 0; i < func.numRodataRelocs; i++) {
162
MOZ_ASSERT(func.rodataRelocs[i] < func.codeSize);
163
masm.addToPCRel4(funcBase + func.rodataRelocs[i], bias);
164
}
165
#else
166
MOZ_CRASH("No jump table support on this platform");
167
#endif
168
}
169
170
masm.flush();
171
if (masm.oom()) {
172
return false;
173
}
174
offsets->end = masm.currentOffset();
175
176
for (size_t i = 0; i < func.numMetadata; i++) {
177
const CraneliftMetadataEntry& metadata = func.metadatas[i];
178
179
CheckedInt<size_t> offset = funcBase;
180
offset += metadata.codeOffset;
181
if (!offset.isValid()) {
182
return false;
183
}
184
185
#ifdef DEBUG
186
// Check code offsets.
187
MOZ_ASSERT(offset.value() >= offsets->normalEntry);
188
MOZ_ASSERT(offset.value() < offsets->ret);
189
190
// Check bytecode offsets.
191
if (metadata.moduleBytecodeOffset > 0 && lineOrBytecode > 0) {
192
MOZ_ASSERT(metadata.moduleBytecodeOffset >= lineOrBytecode);
193
MOZ_ASSERT(metadata.moduleBytecodeOffset <
194
lineOrBytecode + funcBytecodeSize);
195
}
196
#endif
197
// TODO(bug 1532716): Cranelift gives null bytecode offsets for symbolic
198
// accesses.
199
uint32_t bytecodeOffset = metadata.moduleBytecodeOffset
200
? metadata.moduleBytecodeOffset
201
: lineOrBytecode;
202
203
switch (metadata.which) {
204
case CraneliftMetadataEntry::Which::DirectCall: {
205
CallSiteDesc desc(bytecodeOffset, CallSiteDesc::Func);
206
masm.append(desc, CodeOffset(offset.value()), metadata.extra);
207
break;
208
}
209
case CraneliftMetadataEntry::Which::IndirectCall: {
210
CallSiteDesc desc(bytecodeOffset, CallSiteDesc::Dynamic);
211
masm.append(desc, CodeOffset(offset.value()));
212
break;
213
}
214
case CraneliftMetadataEntry::Which::Trap: {
215
Trap trap = (Trap)metadata.extra;
216
BytecodeOffset trapOffset(bytecodeOffset);
217
masm.append(trap, wasm::TrapSite(offset.value(), trapOffset));
218
break;
219
}
220
case CraneliftMetadataEntry::Which::MemoryAccess: {
221
BytecodeOffset trapOffset(bytecodeOffset);
222
masm.appendOutOfBoundsTrap(trapOffset, offset.value());
223
break;
224
}
225
case CraneliftMetadataEntry::Which::SymbolicAccess: {
226
SymbolicAddress sym =
227
ToSymbolicAddress(BD_SymbolicAddress(metadata.extra));
228
masm.append(SymbolicAccess(CodeOffset(offset.value()), sym));
229
break;
230
}
231
default: {
232
MOZ_CRASH("unknown cranelift metadata kind");
233
}
234
}
235
}
236
237
return true;
238
}
239
240
// In Rust, a BatchCompiler variable has a lifetime constrained by those of its
241
// associated StaticEnvironment and ModuleEnvironment. This RAII class ties
242
// them together, as well as makes sure that the compiler is properly destroyed
243
// when it exits scope.
244
245
class AutoCranelift {
246
CraneliftStaticEnvironment staticEnv_;
247
CraneliftModuleEnvironment env_;
248
CraneliftCompiler* compiler_;
249
250
public:
251
explicit AutoCranelift(const ModuleEnvironment& env)
252
: env_(env), compiler_(nullptr) {
253
#ifdef WASM_SUPPORTS_HUGE_MEMORY
254
if (env.hugeMemoryEnabled()) {
255
// In the huge memory configuration, we always reserve the full 4 GB
256
// index space for a heap.
257
staticEnv_.staticMemoryBound = HugeIndexRange;
258
staticEnv_.memoryGuardSize = HugeOffsetGuardLimit;
259
} else {
260
staticEnv_.memoryGuardSize = OffsetGuardLimit;
261
}
262
#endif
263
// Otherwise, heap bounds are stored in the `boundsCheckLimit` field
264
// of TlsData.
265
}
266
bool init() {
267
compiler_ = cranelift_compiler_create(&staticEnv_, &env_);
268
return !!compiler_;
269
}
270
~AutoCranelift() {
271
if (compiler_) {
272
cranelift_compiler_destroy(compiler_);
273
}
274
}
275
operator CraneliftCompiler*() { return compiler_; }
276
};
277
278
CraneliftFuncCompileInput::CraneliftFuncCompileInput(
279
const FuncCompileInput& func)
280
: bytecode(func.begin),
281
bytecodeSize(func.end - func.begin),
282
index(func.index),
283
offset_in_module(func.lineOrBytecode) {}
284
285
static_assert(offsetof(TlsData, boundsCheckLimit) == sizeof(size_t),
286
"fix make_heap() in wasm2clif.rs");
287
288
CraneliftStaticEnvironment::CraneliftStaticEnvironment()
289
:
290
#ifdef JS_CODEGEN_X64
291
hasSse2(Assembler::HasSSE2()),
292
hasSse3(Assembler::HasSSE3()),
293
hasSse41(Assembler::HasSSE41()),
294
hasSse42(Assembler::HasSSE42()),
295
hasPopcnt(Assembler::HasPOPCNT()),
296
hasAvx(Assembler::HasAVX()),
297
hasBmi1(Assembler::HasBMI1()),
298
hasBmi2(Assembler::HasBMI2()),
299
hasLzcnt(Assembler::HasLZCNT()),
300
#else
301
hasSse2(false),
302
hasSse3(false),
303
hasSse41(false),
304
hasSse42(false),
305
hasPopcnt(false),
306
hasAvx(false),
307
hasBmi1(false),
308
hasBmi2(false),
309
hasLzcnt(false),
310
#endif
311
#if defined(XP_WIN)
312
platformIsWindows(true),
313
#else
314
platformIsWindows(false),
315
#endif
316
staticMemoryBound(0),
317
memoryGuardSize(0),
318
memoryBaseTlsOffset(offsetof(TlsData, memoryBase)),
319
instanceTlsOffset(offsetof(TlsData, instance)),
320
interruptTlsOffset(offsetof(TlsData, interrupt)),
321
cxTlsOffset(offsetof(TlsData, cx)),
322
realmCxOffset(JSContext::offsetOfRealm()),
323
realmTlsOffset(offsetof(TlsData, realm)),
324
realmFuncImportTlsOffset(offsetof(FuncImportTls, realm)) {
325
}
326
327
// Most of BaldrMonkey's data structures refer to a "global offset" which is a
328
// byte offset into the `globalArea` field of the `TlsData` struct.
329
//
330
// Cranelift represents global variables with their byte offset from the "VM
331
// context pointer" which is the `WasmTlsReg` pointing to the `TlsData` struct.
332
//
333
// This function translates between the two.
334
335
static size_t globalToTlsOffset(size_t globalOffset) {
336
return offsetof(wasm::TlsData, globalArea) + globalOffset;
337
}
338
339
CraneliftModuleEnvironment::CraneliftModuleEnvironment(
340
const ModuleEnvironment& env)
341
: env(&env), min_memory_length(env.minMemoryLength) {}
342
343
TypeCode env_unpack(BD_ValType valType) {
344
return TypeCode(UnpackTypeCodeType(PackedTypeCode(valType.packed)));
345
}
346
347
bool env_uses_shared_memory(const CraneliftModuleEnvironment* wrapper) {
348
return wrapper->env->usesSharedMemory();
349
}
350
351
const FuncTypeWithId* env_function_signature(
352
const CraneliftModuleEnvironment* wrapper, size_t funcIndex) {
353
return wrapper->env->funcTypes[funcIndex];
354
}
355
356
size_t env_func_import_tls_offset(const CraneliftModuleEnvironment* wrapper,
357
size_t funcIndex) {
358
return globalToTlsOffset(
359
wrapper->env->funcImportGlobalDataOffsets[funcIndex]);
360
}
361
362
bool env_func_is_import(const CraneliftModuleEnvironment* wrapper,
363
size_t funcIndex) {
364
return wrapper->env->funcIsImport(funcIndex);
365
}
366
367
const FuncTypeWithId* env_signature(const CraneliftModuleEnvironment* wrapper,
368
size_t funcTypeIndex) {
369
return &wrapper->env->types[funcTypeIndex].funcType();
370
}
371
372
const TableDesc* env_table(const CraneliftModuleEnvironment* wrapper,
373
size_t tableIndex) {
374
return &wrapper->env->tables[tableIndex];
375
}
376
377
const GlobalDesc* env_global(const CraneliftModuleEnvironment* wrapper,
378
size_t globalIndex) {
379
return &wrapper->env->globals[globalIndex];
380
}
381
382
bool wasm::CraneliftCompileFunctions(const ModuleEnvironment& env,
383
LifoAlloc& lifo,
384
const FuncCompileInputVector& inputs,
385
CompiledCode* code, UniqueChars* error) {
386
MOZ_RELEASE_ASSERT(CraneliftCanCompile());
387
388
MOZ_ASSERT(env.tier() == Tier::Optimized);
389
MOZ_ASSERT(env.optimizedBackend() == OptimizedBackend::Cranelift);
390
MOZ_ASSERT(!env.isAsmJS());
391
392
AutoCranelift compiler(env);
393
if (!compiler.init()) {
394
return false;
395
}
396
397
TempAllocator alloc(&lifo);
398
JitContext jitContext(&alloc);
399
WasmMacroAssembler masm(alloc);
400
MOZ_ASSERT(IsCompilingWasm());
401
402
// Swap in already-allocated empty vectors to avoid malloc/free.
403
MOZ_ASSERT(code->empty());
404
if (!code->swap(masm)) {
405
return false;
406
}
407
408
// Disable instruction spew if we're going to disassemble after code
409
// generation, or the output will be a mess.
410
411
bool jitSpew = JitSpewEnabled(js::jit::JitSpew_Codegen);
412
if (jitSpew) {
413
DisableChannel(js::jit::JitSpew_Codegen);
414
}
415
auto reenableSpew = mozilla::MakeScopeExit([&] {
416
if (jitSpew) {
417
EnableChannel(js::jit::JitSpew_Codegen);
418
}
419
});
420
421
for (const FuncCompileInput& func : inputs) {
422
Decoder d(func.begin, func.end, func.lineOrBytecode, error);
423
424
size_t funcBytecodeSize = func.end - func.begin;
425
if (!ValidateFunctionBody(env, func.index, funcBytecodeSize, d)) {
426
return false;
427
}
428
429
CraneliftFuncCompileInput clifInput(func);
430
431
CraneliftCompiledFunc clifFunc;
432
if (!cranelift_compile_function(compiler, &clifInput, &clifFunc)) {
433
*error = JS_smprintf("Cranelift error in clifFunc #%u", clifInput.index);
434
return false;
435
}
436
437
uint32_t lineOrBytecode = func.lineOrBytecode;
438
const FuncTypeIdDesc& funcTypeId = env.funcTypes[clifInput.index]->id;
439
440
FuncOffsets offsets;
441
if (!GenerateCraneliftCode(masm, clifFunc, funcTypeId, lineOrBytecode,
442
funcBytecodeSize, &offsets)) {
443
return false;
444
}
445
446
if (!code->codeRanges.emplaceBack(func.index, lineOrBytecode, offsets)) {
447
return false;
448
}
449
}
450
451
masm.finish();
452
if (masm.oom()) {
453
return false;
454
}
455
456
if (jitSpew) {
457
// The disassembler uses the jitspew for output, so re-enable now.
458
EnableChannel(js::jit::JitSpew_Codegen);
459
460
uint32_t totalCodeSize = masm.currentOffset();
461
uint8_t* codeBuf = (uint8_t*)js_malloc(totalCodeSize);
462
if (codeBuf) {
463
masm.executableCopy(codeBuf);
464
465
const CodeRangeVector& codeRanges = code->codeRanges;
466
MOZ_ASSERT(codeRanges.length() >= inputs.length());
467
468
// Within the current batch, functions' code ranges have been added in the
469
// same order as the inputs.
470
size_t firstCodeRangeIndex = codeRanges.length() - inputs.length();
471
472
for (size_t i = 0; i < inputs.length(); i++) {
473
int funcIndex = inputs[i].index;
474
mozilla::Unused << funcIndex;
475
476
JitSpew(JitSpew_Codegen, "# ========================================");
477
JitSpew(JitSpew_Codegen, "# Start of wasm cranelift code for index %d",
478
funcIndex);
479
480
size_t codeRangeIndex = firstCodeRangeIndex + i;
481
uint32_t codeStart = codeRanges[codeRangeIndex].begin();
482
uint32_t codeEnd = codeRanges[codeRangeIndex].end();
483
484
jit::Disassemble(
485
codeBuf + codeStart, codeEnd - codeStart,
486
[](const char* text) { JitSpew(JitSpew_Codegen, "%s", text); });
487
488
JitSpew(JitSpew_Codegen, "# End of wasm cranelift code for index %d",
489
funcIndex);
490
}
491
js_free(codeBuf);
492
}
493
}
494
495
return code->swap(masm);
496
}
497
498
////////////////////////////////////////////////////////////////////////////////
499
//
500
// Callbacks from Rust to C++.
501
502
// Offsets assumed by the `make_heap()` function.
503
static_assert(offsetof(wasm::TlsData, memoryBase) == 0, "memory base moved");
504
505
// The translate_call() function in wasm2clif.rs depends on these offsets.
506
static_assert(offsetof(wasm::FuncImportTls, code) == 0,
507
"Import code field moved");
508
static_assert(offsetof(wasm::FuncImportTls, tls) == sizeof(void*),
509
"Import tls moved");
510
511
// Global
512
513
bool global_isConstant(const GlobalDesc* global) {
514
return global->isConstant();
515
}
516
517
bool global_isIndirect(const GlobalDesc* global) {
518
return global->isIndirect();
519
}
520
521
BD_ConstantValue global_constantValue(const GlobalDesc* global) {
522
Val value(global->constantValue());
523
BD_ConstantValue v;
524
v.t = TypeCode(value.type().code());
525
switch (v.t) {
526
case TypeCode::I32:
527
v.u.i32 = value.i32();
528
break;
529
case TypeCode::I64:
530
v.u.i64 = value.i64();
531
break;
532
case TypeCode::F32:
533
v.u.f32 = value.f32();
534
break;
535
case TypeCode::F64:
536
v.u.f64 = value.f64();
537
break;
538
default:
539
MOZ_CRASH("Bad type");
540
}
541
return v;
542
}
543
544
TypeCode global_type(const GlobalDesc* global) {
545
return TypeCode(global->type().code());
546
}
547
548
size_t global_tlsOffset(const GlobalDesc* global) {
549
return globalToTlsOffset(global->offset());
550
}
551
552
// TableDesc
553
554
size_t table_tlsOffset(const TableDesc* table) {
555
MOZ_RELEASE_ASSERT(
556
table->kind == TableKind::FuncRef || table->kind == TableKind::AsmJS,
557
"cranelift doesn't support AnyRef tables yet.");
558
return globalToTlsOffset(table->globalDataOffset);
559
}
560
561
// Sig
562
563
size_t funcType_numArgs(const FuncTypeWithId* funcType) {
564
return funcType->args().length();
565
}
566
567
const BD_ValType* funcType_args(const FuncTypeWithId* funcType) {
568
static_assert(sizeof(BD_ValType) == sizeof(ValType), "update BD_ValType");
569
return (const BD_ValType*)&funcType->args()[0];
570
}
571
572
size_t funcType_numResults(const FuncTypeWithId* funcType) {
573
return funcType->results().length();
574
}
575
576
const BD_ValType* funcType_results(const FuncTypeWithId* funcType) {
577
static_assert(sizeof(BD_ValType) == sizeof(ValType), "update BD_ValType");
578
return (const BD_ValType*)&funcType->results()[0];
579
}
580
581
FuncTypeIdDescKind funcType_idKind(const FuncTypeWithId* funcType) {
582
return funcType->id.kind();
583
}
584
585
size_t funcType_idImmediate(const FuncTypeWithId* funcType) {
586
return funcType->id.immediate();
587
}
588
589
size_t funcType_idTlsOffset(const FuncTypeWithId* funcType) {
590
return globalToTlsOffset(funcType->id.globalDataOffset());
591
}