Source code

Revision control

Copy as Markdown

Other Tools

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: set ts=8 sts=2 et sw=2 tw=80:
*
* Copyright 2015 Mozilla Foundation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "wasm/WasmGenerator.h"
#include "mozilla/CheckedInt.h"
#include "mozilla/EnumeratedRange.h"
#include "mozilla/SHA1.h"
#include <algorithm>
#include "jit/Assembler.h"
#include "jit/JitOptions.h"
#include "js/Printf.h"
#include "util/Memory.h"
#include "util/Text.h"
#include "vm/HelperThreads.h"
#include "vm/Time.h"
#include "vm/TraceLogging.h"
#include "vm/TraceLoggingTypes.h"
#include "wasm/WasmBaselineCompile.h"
#include "wasm/WasmCompile.h"
#include "wasm/WasmCraneliftCompile.h"
#include "wasm/WasmGC.h"
#include "wasm/WasmIonCompile.h"
#include "wasm/WasmStubs.h"
using namespace js;
using namespace js::jit;
using namespace js::wasm;
using mozilla::CheckedInt;
using mozilla::MakeEnumeratedRange;
bool CompiledCode::swap(MacroAssembler& masm) {
MOZ_ASSERT(bytes.empty());
if (!masm.swapBuffer(bytes)) {
return false;
}
callSites.swap(masm.callSites());
callSiteTargets.swap(masm.callSiteTargets());
trapSites.swap(masm.trapSites());
symbolicAccesses.swap(masm.symbolicAccesses());
tryNotes.swap(masm.tryNotes());
codeLabels.swap(masm.codeLabels());
return true;
}
bool CompiledCode::swapCranelift(MacroAssembler& masm,
CraneliftReusableData& data) {
if (!swap(masm)) {
return false;
}
std::swap(data, craneliftReusableData);
return true;
}
// ****************************************************************************
// ModuleGenerator
static const unsigned GENERATOR_LIFO_DEFAULT_CHUNK_SIZE = 4 * 1024;
static const unsigned COMPILATION_LIFO_DEFAULT_CHUNK_SIZE = 64 * 1024;
static const uint32_t BAD_CODE_RANGE = UINT32_MAX;
ModuleGenerator::ModuleGenerator(const CompileArgs& args,
ModuleEnvironment* moduleEnv,
CompilerEnvironment* compilerEnv,
const Atomic<bool>* cancelled,
UniqueChars* error,
UniqueCharsVector* warnings)
: compileArgs_(&args),
error_(error),
warnings_(warnings),
cancelled_(cancelled),
moduleEnv_(moduleEnv),
compilerEnv_(compilerEnv),
linkData_(nullptr),
metadataTier_(nullptr),
lifo_(GENERATOR_LIFO_DEFAULT_CHUNK_SIZE),
masmAlloc_(&lifo_),
masm_(masmAlloc_, *moduleEnv, /* limitedSize= */ false),
debugTrapCodeOffset_(),
lastPatchedCallSite_(0),
startOfUnpatchedCallsites_(0),
parallel_(false),
outstanding_(0),
currentTask_(nullptr),
batchedBytecode_(0),
finishedFuncDefs_(false) {
MOZ_ASSERT(IsCompilingWasm());
}
ModuleGenerator::~ModuleGenerator() {
MOZ_ASSERT_IF(finishedFuncDefs_, !batchedBytecode_);
MOZ_ASSERT_IF(finishedFuncDefs_, !currentTask_);
if (parallel_) {
if (outstanding_) {
AutoLockHelperThreadState lock;
// Remove any pending compilation tasks from the worklist.
size_t removed = RemovePendingWasmCompileTasks(taskState_, mode(), lock);
MOZ_ASSERT(outstanding_ >= removed);
outstanding_ -= removed;
// Wait until all active compilation tasks have finished.
while (true) {
MOZ_ASSERT(outstanding_ >= taskState_.finished().length());
outstanding_ -= taskState_.finished().length();
taskState_.finished().clear();
MOZ_ASSERT(outstanding_ >= taskState_.numFailed());
outstanding_ -= taskState_.numFailed();
taskState_.numFailed() = 0;
if (!outstanding_) {
break;
}
taskState_.condVar().wait(lock); /* failed or finished */
}
}
} else {
MOZ_ASSERT(!outstanding_);
}
// Propagate error state.
if (error_ && !*error_) {
AutoLockHelperThreadState lock;
*error_ = std::move(taskState_.errorMessage());
}
}
bool ModuleGenerator::allocateGlobalBytes(uint32_t bytes, uint32_t align,
uint32_t* globalDataOffset) {
CheckedInt<uint32_t> newGlobalDataLength(metadata_->globalDataLength);
newGlobalDataLength +=
ComputeByteAlignment(newGlobalDataLength.value(), align);
if (!newGlobalDataLength.isValid()) {
return false;
}
*globalDataOffset = newGlobalDataLength.value();
newGlobalDataLength += bytes;
if (!newGlobalDataLength.isValid()) {
return false;
}
metadata_->globalDataLength = newGlobalDataLength.value();
return true;
}
bool ModuleGenerator::init(Metadata* maybeAsmJSMetadata) {
// Perform fallible metadata, linkdata, assumption allocations.
MOZ_ASSERT(isAsmJS() == !!maybeAsmJSMetadata);
if (maybeAsmJSMetadata) {
metadata_ = maybeAsmJSMetadata;
} else {
metadata_ = js_new<Metadata>();
if (!metadata_) {
return false;
}
}
if (compileArgs_->scriptedCaller.filename) {
metadata_->filename =
DuplicateString(compileArgs_->scriptedCaller.filename.get());
if (!metadata_->filename) {
return false;
}
metadata_->filenameIsURL = compileArgs_->scriptedCaller.filenameIsURL;
} else {
MOZ_ASSERT(!compileArgs_->scriptedCaller.filenameIsURL);
}
if (compileArgs_->sourceMapURL) {
metadata_->sourceMapURL = DuplicateString(compileArgs_->sourceMapURL.get());
if (!metadata_->sourceMapURL) {
return false;
}
}
linkData_ = js::MakeUnique<LinkData>(tier());
if (!linkData_) {
return false;
}
metadataTier_ = js::MakeUnique<MetadataTier>(tier());
if (!metadataTier_) {
return false;
}
// funcToCodeRange maps function indices to code-range indices and all
// elements will be initialized by the time module generation is finished.
if (!metadataTier_->funcToCodeRange.appendN(BAD_CODE_RANGE,
moduleEnv_->funcs.length())) {
return false;
}
// Pre-reserve space for large Vectors to avoid the significant cost of the
// final reallocs. In particular, the MacroAssembler can be enormous, so be
// extra conservative. Since large over-reservations may fail when the
// actual allocations will succeed, ignore OOM failures. Note,
// shrinkStorageToFit calls at the end will trim off unneeded capacity.
size_t codeSectionSize =
moduleEnv_->codeSection ? moduleEnv_->codeSection->size : 0;
size_t estimatedCodeSize =
1.2 * EstimateCompiledCodeSize(tier(), codeSectionSize);
(void)masm_.reserve(std::min(estimatedCodeSize, MaxCodeBytesPerProcess));
(void)metadataTier_->codeRanges.reserve(2 * moduleEnv_->numFuncDefs());
const size_t ByteCodesPerCallSite = 50;
(void)metadataTier_->callSites.reserve(codeSectionSize /
ByteCodesPerCallSite);
const size_t ByteCodesPerOOBTrap = 10;
(void)metadataTier_->trapSites[Trap::OutOfBounds].reserve(
codeSectionSize / ByteCodesPerOOBTrap);
// Allocate space in Instance for declarations that need it.
MOZ_ASSERT(metadata_->globalDataLength == 0);
for (size_t i = 0; i < moduleEnv_->funcImportGlobalDataOffsets.length();
i++) {
uint32_t globalDataOffset;
if (!allocateGlobalBytes(sizeof(FuncImportInstanceData), sizeof(void*),
&globalDataOffset)) {
return false;
}
moduleEnv_->funcImportGlobalDataOffsets[i] = globalDataOffset;
FuncType copy;
if (!copy.clone(*moduleEnv_->funcs[i].type)) {
return false;
}
if (!metadataTier_->funcImports.emplaceBack(std::move(copy),
globalDataOffset)) {
return false;
}
}
for (TableDesc& table : moduleEnv_->tables) {
if (!allocateGlobalBytes(sizeof(TableInstanceData), sizeof(void*),
&table.globalDataOffset)) {
return false;
}
}
for (TagDesc& tag : moduleEnv_->tags) {
if (!allocateGlobalBytes(sizeof(WasmTagObject*), sizeof(void*),
&tag.globalDataOffset)) {
return false;
}
}
if (!isAsmJS()) {
// Copy type definitions to metadata that are required at runtime,
// allocating global data so that codegen can find the type id's at
// runtime.
for (uint32_t typeIndex = 0; typeIndex < moduleEnv_->types->length();
typeIndex++) {
const TypeDef& typeDef = (*moduleEnv_->types)[typeIndex];
TypeIdDesc& typeId = moduleEnv_->typeIds[typeIndex];
if (TypeIdDesc::isGlobal(typeDef)) {
uint32_t globalDataOffset;
if (!allocateGlobalBytes(sizeof(void*), sizeof(void*),
&globalDataOffset)) {
return false;
}
typeId = TypeIdDesc::global(typeDef, globalDataOffset);
TypeDef copy;
if (!copy.clone(typeDef)) {
return false;
}
if (!metadata_->types.emplaceBack(std::move(copy), typeId)) {
return false;
}
} else {
typeId = TypeIdDesc::immediate(typeDef);
}
}
// If we allow type indices, then we need to rewrite the index space to
// account for types that are omitted from metadata, such as function
// types that fit in an immediate.
if (moduleEnv_->functionReferencesEnabled()) {
// Do a linear pass to create a map from src index to dest index.
RenumberVector renumbering;
if (!renumbering.reserve(moduleEnv_->types->length())) {
return false;
}
for (uint32_t srcIndex = 0, destIndex = 0;
srcIndex < moduleEnv_->types->length(); srcIndex++) {
const TypeDef& typeDef = (*moduleEnv_->types)[srcIndex];
if (!TypeIdDesc::isGlobal(typeDef)) {
renumbering.infallibleAppend(UINT32_MAX);
continue;
}
MOZ_ASSERT(renumbering.length() == srcIndex);
renumbering.infallibleAppend(destIndex++);
}
// Apply the renumbering
for (TypeDefWithId& typeDef : metadata_->types) {
typeDef.renumber(renumbering);
}
metadata_->typesRenumbering = std::move(renumbering);
}
}
for (GlobalDesc& global : moduleEnv_->globals) {
if (global.isConstant()) {
continue;
}
uint32_t width =
global.isIndirect() ? sizeof(void*) : SizeOf(global.type());
uint32_t globalDataOffset;
if (!allocateGlobalBytes(width, width, &globalDataOffset)) {
return false;
}
global.setOffset(globalDataOffset);
}
// Accumulate all exported functions:
// - explicitly marked as such;
// - implicitly exported by being an element of function tables;
// - implicitly exported by being the start function;
// - implicitly exported by being used in global ref.func initializer
// ModuleEnvironment accumulates this information for us during decoding,
// transfer it to the FuncExportVector stored in Metadata.
uint32_t exportedFuncCount = 0;
for (const FuncDesc& func : moduleEnv_->funcs) {
if (func.isExported()) {
exportedFuncCount++;
}
}
if (!metadataTier_->funcExports.reserve(exportedFuncCount)) {
return false;
}
for (uint32_t funcIndex = 0; funcIndex < moduleEnv_->funcs.length();
funcIndex++) {
const FuncDesc& func = moduleEnv_->funcs[funcIndex];
if (!func.isExported()) {
continue;
}
FuncType funcType;
if (!funcType.clone(*func.type)) {
return false;
}
metadataTier_->funcExports.infallibleEmplaceBack(std::move(funcType),
funcIndex, func.isEager());
}
// Determine whether parallel or sequential compilation is to be used and
// initialize the CompileTasks that will be used in either mode.
MOZ_ASSERT(GetHelperThreadCount() > 1);
uint32_t numTasks;
if (CanUseExtraThreads() && GetHelperThreadCPUCount() > 1) {
parallel_ = true;
numTasks = 2 * GetMaxWasmCompilationThreads();
} else {
numTasks = 1;
}
if (!tasks_.initCapacity(numTasks)) {
return false;
}
for (size_t i = 0; i < numTasks; i++) {
tasks_.infallibleEmplaceBack(*moduleEnv_, *compilerEnv_, taskState_,
COMPILATION_LIFO_DEFAULT_CHUNK_SIZE);
}
if (!freeTasks_.reserve(numTasks)) {
return false;
}
for (size_t i = 0; i < numTasks; i++) {
freeTasks_.infallibleAppend(&tasks_[i]);
}
// Fill in function stubs for each import so that imported functions can be
// used in all the places that normal function definitions can (table
// elements, export calls, etc).
CompiledCode& importCode = tasks_[0].output;
MOZ_ASSERT(importCode.empty());
if (!GenerateImportFunctions(*moduleEnv_, metadataTier_->funcImports,
&importCode)) {
return false;
}
if (!linkCompiledCode(importCode)) {
return false;
}
importCode.clear();
return true;
}
bool ModuleGenerator::funcIsCompiled(uint32_t funcIndex) const {
return metadataTier_->funcToCodeRange[funcIndex] != BAD_CODE_RANGE;
}
const CodeRange& ModuleGenerator::funcCodeRange(uint32_t funcIndex) const {
MOZ_ASSERT(funcIsCompiled(funcIndex));
const CodeRange& cr =
metadataTier_->codeRanges[metadataTier_->funcToCodeRange[funcIndex]];
MOZ_ASSERT(cr.isFunction());
return cr;
}
static bool InRange(uint32_t caller, uint32_t callee) {
// We assume JumpImmediateRange is defined conservatively enough that the
// slight difference between 'caller' (which is really the return address
// offset) and the actual base of the relative displacement computation
// isn't significant.
uint32_t range = std::min(JitOptions.jumpThreshold, JumpImmediateRange);
if (caller < callee) {
return callee - caller < range;
}
return caller - callee < range;
}
using OffsetMap =
HashMap<uint32_t, uint32_t, DefaultHasher<uint32_t>, SystemAllocPolicy>;
using TrapMaybeOffsetArray =
EnumeratedArray<Trap, Trap::Limit, Maybe<uint32_t>>;
bool ModuleGenerator::linkCallSites() {
AutoCreatedBy acb(masm_, "linkCallSites");
masm_.haltingAlign(CodeAlignment);
// Create far jumps for calls that have relative offsets that may otherwise
// go out of range. This method is called both between function bodies (at a
// frequency determined by the ISA's jump range) and once at the very end of
// a module's codegen after all possible calls/traps have been emitted.
OffsetMap existingCallFarJumps;
for (; lastPatchedCallSite_ < metadataTier_->callSites.length();
lastPatchedCallSite_++) {
const CallSite& callSite = metadataTier_->callSites[lastPatchedCallSite_];
const CallSiteTarget& target = callSiteTargets_[lastPatchedCallSite_];
uint32_t callerOffset = callSite.returnAddressOffset();
switch (callSite.kind()) {
case CallSiteDesc::Import:
case CallSiteDesc::Indirect:
case CallSiteDesc::IndirectFast:
case CallSiteDesc::Symbolic:
case CallSiteDesc::Breakpoint:
case CallSiteDesc::EnterFrame:
case CallSiteDesc::LeaveFrame:
break;
case CallSiteDesc::Func: {
if (funcIsCompiled(target.funcIndex())) {
uint32_t calleeOffset =
funcCodeRange(target.funcIndex()).funcUncheckedCallEntry();
if (InRange(callerOffset, calleeOffset)) {
masm_.patchCall(callerOffset, calleeOffset);
break;
}
}
OffsetMap::AddPtr p =
existingCallFarJumps.lookupForAdd(target.funcIndex());
if (!p) {
Offsets offsets;
offsets.begin = masm_.currentOffset();
if (!callFarJumps_.emplaceBack(target.funcIndex(),
masm_.farJumpWithPatch())) {
return false;
}
offsets.end = masm_.currentOffset();
if (masm_.oom()) {
return false;
}
if (!metadataTier_->codeRanges.emplaceBack(CodeRange::FarJumpIsland,
offsets)) {
return false;
}
if (!existingCallFarJumps.add(p, target.funcIndex(), offsets.begin)) {
return false;
}
}
masm_.patchCall(callerOffset, p->value());
break;
}
}
}
masm_.flushBuffer();
return !masm_.oom();
}
void ModuleGenerator::noteCodeRange(uint32_t codeRangeIndex,
const CodeRange& codeRange) {
switch (codeRange.kind()) {
case CodeRange::Function:
MOZ_ASSERT(metadataTier_->funcToCodeRange[codeRange.funcIndex()] ==
BAD_CODE_RANGE);
metadataTier_->funcToCodeRange[codeRange.funcIndex()] = codeRangeIndex;
break;
case CodeRange::InterpEntry:
metadataTier_->lookupFuncExport(codeRange.funcIndex())
.initEagerInterpEntryOffset(codeRange.begin());
break;
case CodeRange::JitEntry:
// Nothing to do: jit entries are linked in the jump tables.
break;
case CodeRange::ImportJitExit:
metadataTier_->funcImports[codeRange.funcIndex()].initJitExitOffset(
codeRange.begin());
break;
case CodeRange::ImportInterpExit:
metadataTier_->funcImports[codeRange.funcIndex()].initInterpExitOffset(
codeRange.begin());
break;
case CodeRange::DebugTrap:
MOZ_ASSERT(!debugTrapCodeOffset_);
debugTrapCodeOffset_ = codeRange.begin();
break;
case CodeRange::TrapExit:
MOZ_ASSERT(!linkData_->trapOffset);
linkData_->trapOffset = codeRange.begin();
break;
case CodeRange::Throw:
// Jumped to by other stubs, so nothing to do.
break;
case CodeRange::FarJumpIsland:
case CodeRange::BuiltinThunk:
MOZ_CRASH("Unexpected CodeRange kind");
}
}
// Append every element from `srcVec` where `filterOp(srcElem) == true`.
// Applies `mutateOp(dstElem)` to every element that is appended.
template <class Vec, class FilterOp, class MutateOp>
static bool AppendForEach(Vec* dstVec, const Vec& srcVec, FilterOp filterOp,
MutateOp mutateOp) {
// Eagerly grow the vector to the whole src vector. Any filtered elements
// will be trimmed later.
if (!dstVec->growByUninitialized(srcVec.length())) {
return false;
}
using T = typename Vec::ElementType;
T* dstBegin = dstVec->begin();
T* dstEnd = dstVec->end();
// We appended srcVec.length() elements at the beginning, so we append
// elements starting at the first uninitialized element.
T* dst = dstEnd - srcVec.length();
for (const T* src = srcVec.begin(); src != srcVec.end(); src++) {
if (!filterOp(src)) {
continue;
}
new (dst) T(*src);
mutateOp(dst - dstBegin, dst);
dst++;
}
// Trim off the filtered out elements that were eagerly added at the
// beginning
size_t newSize = dst - dstBegin;
if (newSize != dstVec->length()) {
dstVec->shrinkTo(newSize);
dstVec->shrinkStorageToFit();
}
return true;
}
template <typename T>
bool FilterNothing(const T* element) {
return true;
}
// The same as the above `AppendForEach`, without performing any filtering.
template <class Vec, class MutateOp>
static bool AppendForEach(Vec* dstVec, const Vec& srcVec, MutateOp mutateOp) {
using T = typename Vec::ElementType;
return AppendForEach(dstVec, srcVec, &FilterNothing<T>, mutateOp);
}
bool ModuleGenerator::linkCompiledCode(CompiledCode& code) {
AutoCreatedBy acb(masm_, "ModuleGenerator::linkCompiledCode");
// Before merging in new code, if calls in a prior code range might go out of
// range, insert far jumps to extend the range.
if (!InRange(startOfUnpatchedCallsites_,
masm_.size() + code.bytes.length())) {
startOfUnpatchedCallsites_ = masm_.size();
if (!linkCallSites()) {
return false;
}
}
// All code offsets in 'code' must be incremented by their position in the
// overall module when the code was appended.
masm_.haltingAlign(CodeAlignment);
const size_t offsetInModule = masm_.size();
if (!masm_.appendRawCode(code.bytes.begin(), code.bytes.length())) {
return false;
}
auto codeRangeOp = [offsetInModule, this](uint32_t codeRangeIndex,
CodeRange* codeRange) {
codeRange->offsetBy(offsetInModule);
noteCodeRange(codeRangeIndex, *codeRange);
};
if (!AppendForEach(&metadataTier_->codeRanges, code.codeRanges,
codeRangeOp)) {
return false;
}
auto callSiteOp = [=](uint32_t, CallSite* cs) {
cs->offsetBy(offsetInModule);
};
if (!AppendForEach(&metadataTier_->callSites, code.callSites, callSiteOp)) {
return false;
}
if (!callSiteTargets_.appendAll(code.callSiteTargets)) {
return false;
}
for (Trap trap : MakeEnumeratedRange(Trap::Limit)) {
auto trapSiteOp = [=](uint32_t, TrapSite* ts) {
ts->offsetBy(offsetInModule);
};
if (!AppendForEach(&metadataTier_->trapSites[trap], code.trapSites[trap],
trapSiteOp)) {
return false;
}
}
for (const SymbolicAccess& access : code.symbolicAccesses) {
uint32_t patchAt = offsetInModule + access.patchAt.offset();
if (!linkData_->symbolicLinks[access.target].append(patchAt)) {
return false;
}
}
for (const CodeLabel& codeLabel : code.codeLabels) {
LinkData::InternalLink link;
link.patchAtOffset = offsetInModule + codeLabel.patchAt().offset();
link.targetOffset = offsetInModule + codeLabel.target().offset();
#ifdef JS_CODELABEL_LINKMODE
link.mode = codeLabel.linkMode();
#endif
if (!linkData_->internalLinks.append(link)) {
return false;
}
}
for (size_t i = 0; i < code.stackMaps.length(); i++) {
StackMaps::Maplet maplet = code.stackMaps.move(i);
maplet.offsetBy(offsetInModule);
if (!metadataTier_->stackMaps.add(maplet)) {
// This function is now the only owner of maplet.map, so we'd better
// free it right now.
maplet.map->destroy();
return false;
}
}
auto tryNoteFilter = [](const WasmTryNote* tn) {
// Filter out all try notes that were never given a try body. This may
// happen due to dead code elimination.
return tn->hasTryBody();
};
auto tryNoteOp = [=](uint32_t, WasmTryNote* tn) {
tn->offsetBy(offsetInModule);
};
if (!AppendForEach(&metadataTier_->tryNotes, code.tryNotes, tryNoteFilter,
tryNoteOp)) {
return false;
}
return true;
}
static bool ExecuteCompileTask(CompileTask* task, UniqueChars* error) {
MOZ_ASSERT(task->lifo.isEmpty());
MOZ_ASSERT(task->output.empty());
switch (task->compilerEnv.tier()) {
case Tier::Optimized:
switch (task->compilerEnv.optimizedBackend()) {
case OptimizedBackend::Cranelift:
if (!CraneliftCompileFunctions(task->moduleEnv, task->compilerEnv,
task->lifo, task->inputs,
&task->output, error)) {
return false;
}
break;
case OptimizedBackend::Ion:
if (!IonCompileFunctions(task->moduleEnv, task->compilerEnv,
task->lifo, task->inputs, &task->output,
error)) {
return false;
}
break;
}
break;
case Tier::Baseline:
if (!BaselineCompileFunctions(task->moduleEnv, task->compilerEnv,
task->lifo, task->inputs, &task->output,
error)) {
return false;
}
break;
}
MOZ_ASSERT(task->lifo.isEmpty());
MOZ_ASSERT(task->inputs.length() == task->output.codeRanges.length());
task->inputs.clear();
return true;
}
void CompileTask::runHelperThreadTask(AutoLockHelperThreadState& lock) {
TraceLoggerThread* logger = TraceLoggerForCurrentThread();
AutoTraceLog logCompile(logger, TraceLogger_WasmCompilation);
UniqueChars error;
bool ok;
{
AutoUnlockHelperThreadState unlock(lock);
ok = ExecuteCompileTask(this, &error);
}
// Don't release the lock between updating our state and returning from this
// method.
if (!ok || !state.finished().append(this)) {
state.numFailed()++;
if (!state.errorMessage()) {
state.errorMessage() = std::move(error);
}
}
state.condVar().notify_one(); /* failed or finished */
}
ThreadType CompileTask::threadType() {
switch (compilerEnv.mode()) {
case CompileMode::Once:
case CompileMode::Tier1:
return ThreadType::THREAD_TYPE_WASM_COMPILE_TIER1;
case CompileMode::Tier2:
return ThreadType::THREAD_TYPE_WASM_COMPILE_TIER2;
default:
MOZ_CRASH();
}
}
bool ModuleGenerator::locallyCompileCurrentTask() {
if (!ExecuteCompileTask(currentTask_, error_)) {
return false;
}
if (!finishTask(currentTask_)) {
return false;
}
currentTask_ = nullptr;
batchedBytecode_ = 0;
return true;
}
bool ModuleGenerator::finishTask(CompileTask* task) {
AutoCreatedBy acb(masm_, "ModuleGenerator::finishTask");
masm_.haltingAlign(CodeAlignment);
if (!linkCompiledCode(task->output)) {
return false;
}
task->output.clear();
MOZ_ASSERT(task->inputs.empty());
MOZ_ASSERT(task->output.empty());
MOZ_ASSERT(task->lifo.isEmpty());
freeTasks_.infallibleAppend(task);
return true;
}
bool ModuleGenerator::launchBatchCompile() {
MOZ_ASSERT(currentTask_);
if (cancelled_ && *cancelled_) {
return false;
}
if (!parallel_) {
return locallyCompileCurrentTask();
}
if (!StartOffThreadWasmCompile(currentTask_, mode())) {
return false;
}
outstanding_++;
currentTask_ = nullptr;
batchedBytecode_ = 0;
return true;
}
bool ModuleGenerator::finishOutstandingTask() {
MOZ_ASSERT(parallel_);
CompileTask* task = nullptr;
{
AutoLockHelperThreadState lock;
while (true) {
MOZ_ASSERT(outstanding_ > 0);
if (taskState_.numFailed() > 0) {
return false;
}
if (!taskState_.finished().empty()) {
outstanding_--;
task = taskState_.finished().popCopy();
break;
}
taskState_.condVar().wait(lock); /* failed or finished */
}
}
// Call outside of the compilation lock.
return finishTask(task);
}
bool ModuleGenerator::compileFuncDef(uint32_t funcIndex,
uint32_t lineOrBytecode,
const uint8_t* begin, const uint8_t* end,
Uint32Vector&& lineNums) {
MOZ_ASSERT(!finishedFuncDefs_);
MOZ_ASSERT(funcIndex < moduleEnv_->numFuncs());
uint32_t threshold;
switch (tier()) {
case Tier::Baseline:
threshold = JitOptions.wasmBatchBaselineThreshold;
break;
case Tier::Optimized:
switch (compilerEnv_->optimizedBackend()) {
case OptimizedBackend::Ion:
threshold = JitOptions.wasmBatchIonThreshold;
break;
case OptimizedBackend::Cranelift:
threshold = JitOptions.wasmBatchCraneliftThreshold;
break;
default:
MOZ_CRASH("Invalid optimizedBackend value");
}
break;
default:
MOZ_CRASH("Invalid tier value");
break;
}
uint32_t funcBytecodeLength = end - begin;
// Do not go over the threshold if we can avoid it: spin off the compilation
// before appending the function if we would go over. (Very large single
// functions may still exceed the threshold but this is fine; it'll be very
// uncommon and is in any case safely handled by the MacroAssembler's buffer
// limit logic.)
if (currentTask_ && currentTask_->inputs.length() &&
batchedBytecode_ + funcBytecodeLength > threshold) {
if (!launchBatchCompile()) {
return false;
}
}
if (!currentTask_) {
if (freeTasks_.empty() && !finishOutstandingTask()) {
return false;
}
currentTask_ = freeTasks_.popCopy();
}
if (!currentTask_->inputs.emplaceBack(funcIndex, lineOrBytecode, begin, end,
std::move(lineNums))) {
return false;
}
batchedBytecode_ += funcBytecodeLength;
MOZ_ASSERT(batchedBytecode_ <= MaxCodeSectionBytes);
return true;
}
bool ModuleGenerator::finishFuncDefs() {
MOZ_ASSERT(!finishedFuncDefs_);
if (currentTask_ && !locallyCompileCurrentTask()) {
return false;
}
finishedFuncDefs_ = true;
return true;
}
bool ModuleGenerator::finishCodegen() {
// Now that all functions and stubs are generated and their CodeRanges
// known, patch all calls (which can emit far jumps) and far jumps. Linking
// can emit tiny far-jump stubs, so there is an ordering dependency here.
if (!linkCallSites()) {
return false;
}
for (CallFarJump far : callFarJumps_) {
masm_.patchFarJump(far.jump,
funcCodeRange(far.funcIndex).funcUncheckedCallEntry());
}
metadataTier_->debugTrapOffset = debugTrapCodeOffset_;
// None of the linking or far-jump operations should emit masm metadata.
MOZ_ASSERT(masm_.callSites().empty());
MOZ_ASSERT(masm_.callSiteTargets().empty());
MOZ_ASSERT(masm_.trapSites().empty());
MOZ_ASSERT(masm_.symbolicAccesses().empty());
MOZ_ASSERT(masm_.tryNotes().empty());
MOZ_ASSERT(masm_.codeLabels().empty());
masm_.finish();
return !masm_.oom();
}
bool ModuleGenerator::finishMetadataTier() {
// The stackmaps aren't yet sorted. Do so now, since we'll need to
// binary-search them at GC time.
metadataTier_->stackMaps.finishAndSort();
// The try notes also need to be sorted to simplify lookup.
std::sort(metadataTier_->tryNotes.begin(), metadataTier_->tryNotes.end());
#ifdef DEBUG
// Check that the stackmap contains no duplicates, since that could lead to
// ambiguities about stack slot pointerness.
const uint8_t* previousNextInsnAddr = nullptr;
for (size_t i = 0; i < metadataTier_->stackMaps.length(); i++) {
const StackMaps::Maplet& maplet = metadataTier_->stackMaps.get(i);
MOZ_ASSERT_IF(i > 0, uintptr_t(maplet.nextInsnAddr) >
uintptr_t(previousNextInsnAddr));
previousNextInsnAddr = maplet.nextInsnAddr;
}
// Assert all sorted metadata is sorted.
uint32_t last = 0;
for (const CodeRange& codeRange : metadataTier_->codeRanges) {
MOZ_ASSERT(codeRange.begin() >= last);
last = codeRange.end();
}
last = 0;
for (const CallSite& callSite : metadataTier_->callSites) {
MOZ_ASSERT(callSite.returnAddressOffset() >= last);
last = callSite.returnAddressOffset();
}
for (Trap trap : MakeEnumeratedRange(Trap::Limit)) {
last = 0;
for (const TrapSite& trapSite : metadataTier_->trapSites[trap]) {
MOZ_ASSERT(trapSite.pcOffset >= last);
last = trapSite.pcOffset;
}
}
// Try notes should be sorted so that the end of ranges are in rising order
// so that the innermost catch handler is chosen.
last = 0;
for (const WasmTryNote& tryNote : metadataTier_->tryNotes) {
MOZ_ASSERT(tryNote.tryBodyEnd() >= last);
MOZ_ASSERT(tryNote.tryBodyEnd() > tryNote.tryBodyBegin());
last = tryNote.tryBodyBegin();
}
#endif
// These Vectors can get large and the excess capacity can be significant,
// so realloc them down to size.
metadataTier_->funcToCodeRange.shrinkStorageToFit();
metadataTier_->codeRanges.shrinkStorageToFit();
metadataTier_->callSites.shrinkStorageToFit();
metadataTier_->trapSites.shrinkStorageToFit();
metadataTier_->tryNotes.shrinkStorageToFit();
for (Trap trap : MakeEnumeratedRange(Trap::Limit)) {
metadataTier_->trapSites[trap].shrinkStorageToFit();
}
return true;
}
UniqueCodeTier ModuleGenerator::finishCodeTier() {
MOZ_ASSERT(finishedFuncDefs_);
while (outstanding_ > 0) {
if (!finishOutstandingTask()) {
return nullptr;
}
}
#ifdef DEBUG
for (uint32_t codeRangeIndex : metadataTier_->funcToCodeRange) {
MOZ_ASSERT(codeRangeIndex != BAD_CODE_RANGE);
}
#endif
// Now that all imports/exports are known, we can generate a special
// CompiledCode containing stubs.
CompiledCode& stubCode = tasks_[0].output;
MOZ_ASSERT(stubCode.empty());
if (!GenerateStubs(*moduleEnv_, metadataTier_->funcImports,
metadataTier_->funcExports, &stubCode)) {
return nullptr;
}
if (!linkCompiledCode(stubCode)) {
return nullptr;
}
// Finish linking and metadata.
if (!finishCodegen()) {
return nullptr;
}
if (!finishMetadataTier()) {
return nullptr;
}
UniqueModuleSegment segment =
ModuleSegment::create(tier(), masm_, *linkData_);
if (!segment) {
warnf("failed to allocate executable memory for module");
return nullptr;
}
metadataTier_->stackMaps.offsetBy(uintptr_t(segment->base()));
#ifdef DEBUG
// Check that each stackmap is associated with a plausible instruction.
for (size_t i = 0; i < metadataTier_->stackMaps.length(); i++) {
MOZ_ASSERT(IsValidStackMapKey(compilerEnv_->debugEnabled(),
metadataTier_->stackMaps.get(i).nextInsnAddr),
"wasm stackmap does not reference a valid insn");
}
#endif
return js::MakeUnique<CodeTier>(std::move(metadataTier_), std::move(segment));
}
SharedMetadata ModuleGenerator::finishMetadata(const Bytes& bytecode) {
// Finish initialization of Metadata, which is only needed for constructing
// the initial Module, not for tier-2 compilation.
MOZ_ASSERT(mode() != CompileMode::Tier2);
// Copy over data from the ModuleEnvironment.
metadata_->memory = moduleEnv_->memory;
metadata_->startFuncIndex = moduleEnv_->startFuncIndex;
metadata_->tables = std::move(moduleEnv_->tables);
metadata_->globals = std::move(moduleEnv_->globals);
metadata_->tags = std::move(moduleEnv_->tags);
metadata_->nameCustomSectionIndex = moduleEnv_->nameCustomSectionIndex;
metadata_->moduleName = moduleEnv_->moduleName;
metadata_->funcNames = std::move(moduleEnv_->funcNames);
metadata_->omitsBoundsChecks = moduleEnv_->hugeMemoryEnabled();
// Copy over additional debug information.
if (compilerEnv_->debugEnabled()) {
metadata_->debugEnabled = true;
const size_t numFuncs = moduleEnv_->funcs.length();
if (!metadata_->debugFuncArgTypes.resize(numFuncs)) {
return nullptr;
}
if (!metadata_->debugFuncReturnTypes.resize(numFuncs)) {
return nullptr;
}
for (size_t i = 0; i < numFuncs; i++) {
if (!metadata_->debugFuncArgTypes[i].appendAll(
moduleEnv_->funcs[i].type->args())) {
return nullptr;
}
if (!metadata_->debugFuncReturnTypes[i].appendAll(
moduleEnv_->funcs[i].type->results())) {
return nullptr;
}
}
static_assert(sizeof(ModuleHash) <= sizeof(mozilla::SHA1Sum::Hash),
"The ModuleHash size shall not exceed the SHA1 hash size.");
mozilla::SHA1Sum::Hash hash;
mozilla::SHA1Sum sha1Sum;
sha1Sum.update(bytecode.begin(), bytecode.length());
sha1Sum.finish(hash);
memcpy(metadata_->debugHash, hash, sizeof(ModuleHash));
}
MOZ_ASSERT_IF(moduleEnv_->nameCustomSectionIndex, !!metadata_->namePayload);
// Metadata shouldn't be mutably modified after finishMetadata().
SharedMetadata metadata = metadata_;
metadata_ = nullptr;
return metadata;
}
SharedModule ModuleGenerator::finishModule(
const ShareableBytes& bytecode,
JS::OptimizedEncodingListener* maybeTier2Listener) {
MOZ_ASSERT(mode() == CompileMode::Once || mode() == CompileMode::Tier1);
UniqueCodeTier codeTier = finishCodeTier();
if (!codeTier) {
return nullptr;
}
JumpTables jumpTables;
if (!jumpTables.init(mode(), codeTier->segment(),
codeTier->metadata().codeRanges)) {
return nullptr;
}
// Copy over data from the Bytecode, which is going away at the end of
// compilation.
DataSegmentVector dataSegments;
if (!dataSegments.reserve(moduleEnv_->dataSegments.length())) {
return nullptr;
}
for (const DataSegmentEnv& srcSeg : moduleEnv_->dataSegments) {
MutableDataSegment dstSeg = js_new<DataSegment>();
if (!dstSeg) {
return nullptr;
}
if (!dstSeg->init(bytecode, srcSeg)) {
return nullptr;
}
dataSegments.infallibleAppend(std::move(dstSeg));
}
CustomSectionVector customSections;
if (!customSections.reserve(moduleEnv_->customSections.length())) {
return nullptr;
}
for (const CustomSectionEnv& srcSec : moduleEnv_->customSections) {
CustomSection sec;
if (!sec.name.append(bytecode.begin() + srcSec.nameOffset,
srcSec.nameLength)) {
return nullptr;
}
MutableBytes payload = js_new<ShareableBytes>();
if (!payload) {
return nullptr;
}
if (!payload->append(bytecode.begin() + srcSec.payloadOffset,
srcSec.payloadLength)) {
return nullptr;
}
sec.payload = std::move(payload);
customSections.infallibleAppend(std::move(sec));
}
if (moduleEnv_->nameCustomSectionIndex) {
metadata_->namePayload =
customSections[*moduleEnv_->nameCustomSectionIndex].payload;
}
SharedMetadata metadata = finishMetadata(bytecode.bytes);
if (!metadata) {
return nullptr;
}
MutableCode code =
js_new<Code>(std::move(codeTier), *metadata, std::move(jumpTables));
if (!code || !code->initialize(*linkData_)) {
return nullptr;
}
const ShareableBytes* debugBytecode = nullptr;
if (compilerEnv_->debugEnabled()) {
MOZ_ASSERT(mode() == CompileMode::Once);
MOZ_ASSERT(tier() == Tier::Debug);
debugBytecode = &bytecode;
}
// All the components are finished, so create the complete Module and start
// tier-2 compilation if requested.
MutableModule module = js_new<Module>(
*code, std::move(moduleEnv_->imports), std::move(moduleEnv_->exports),
std::move(dataSegments), std::move(moduleEnv_->elemSegments),
std::move(customSections), debugBytecode);
if (!module) {
return nullptr;
}
if (!isAsmJS() && compileArgs_->features.testSerialization) {
MOZ_RELEASE_ASSERT(mode() == CompileMode::Once &&
tier() == Tier::Serialized);
Bytes serializedBytes;
if (!module->serialize(*linkData_, &serializedBytes)) {
return nullptr;
}
MutableModule deserializedModule =
Module::deserialize(serializedBytes.begin(), serializedBytes.length());
if (!deserializedModule) {
return nullptr;
}
module = deserializedModule;
// Perform storeOptimizedEncoding here instead of below so we don't have to
// re-serialize the module.
if (maybeTier2Listener) {
maybeTier2Listener->storeOptimizedEncoding(serializedBytes.begin(),
serializedBytes.length());
maybeTier2Listener = nullptr;
}
}
if (mode() == CompileMode::Tier1) {
module->startTier2(*compileArgs_, bytecode, maybeTier2Listener);
} else if (tier() == Tier::Serialized && maybeTier2Listener) {
Bytes bytes;
if (module->serialize(*linkData_, &bytes)) {
maybeTier2Listener->storeOptimizedEncoding(bytes.begin(), bytes.length());
}
}
return module;
}
bool ModuleGenerator::finishTier2(const Module& module) {
MOZ_ASSERT(mode() == CompileMode::Tier2);
MOZ_ASSERT(tier() == Tier::Optimized);
MOZ_ASSERT(!compilerEnv_->debugEnabled());
if (cancelled_ && *cancelled_) {
return false;
}
UniqueCodeTier codeTier = finishCodeTier();
if (!codeTier) {
return false;
}
if (MOZ_UNLIKELY(JitOptions.wasmDelayTier2)) {
// Introduce an artificial delay when testing wasmDelayTier2, since we
// want to exercise both tier1 and tier2 code in this case.
ThisThread::SleepMilliseconds(500);
}
return module.finishTier2(*linkData_, std::move(codeTier));
}
void ModuleGenerator::warnf(const char* msg, ...) {
if (!warnings_) {
return;
}
va_list ap;
va_start(ap, msg);
UniqueChars str(JS_vsmprintf(msg, ap));
va_end(ap);
if (!str) {
return;
}
(void)warnings_->append(std::move(str));
}
size_t CompiledCode::sizeOfExcludingThis(
mozilla::MallocSizeOf mallocSizeOf) const {
size_t trapSitesSize = 0;
for (const TrapSiteVector& vec : trapSites) {
trapSitesSize += vec.sizeOfExcludingThis(mallocSizeOf);
}
return bytes.sizeOfExcludingThis(mallocSizeOf) +
codeRanges.sizeOfExcludingThis(mallocSizeOf) +
callSites.sizeOfExcludingThis(mallocSizeOf) +
callSiteTargets.sizeOfExcludingThis(mallocSizeOf) + trapSitesSize +
symbolicAccesses.sizeOfExcludingThis(mallocSizeOf) +
tryNotes.sizeOfExcludingThis(mallocSizeOf) +
codeLabels.sizeOfExcludingThis(mallocSizeOf);
}
size_t CompileTask::sizeOfExcludingThis(
mozilla::MallocSizeOf mallocSizeOf) const {
return lifo.sizeOfExcludingThis(mallocSizeOf) +
inputs.sizeOfExcludingThis(mallocSizeOf) +
output.sizeOfExcludingThis(mallocSizeOf);
}