firefox-main/js/src/jit/WarpBuilder.cpp (file symbol)

Source code

Revision control

Copy as Markdown

Other Tools

/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "jit/WarpBuilder.h"
#include "mozilla/DebugOnly.h"
#include "jit/BaselineFrame.h"
#include "jit/CacheIR.h"
#include "jit/CompileInfo.h"
#include "jit/InlineScriptTree.h"
#include "jit/MIR-wasm.h"
#include "jit/MIR.h"
#include "jit/MIRGenerator.h"
#include "jit/MIRGraph.h"
#include "jit/WarpCacheIRTranspiler.h"
#include "jit/WarpSnapshot.h"
#include "js/friend/ErrorMessages.h" // JSMSG_BAD_CONST_ASSIGN
#include "vm/ConstantCompareOperand.h"
#include "vm/GeneratorObject.h"
#include "vm/Interpreter.h"
#include "vm/Opcodes.h"
#include "vm/TypeofEqOperand.h" // TypeofEqOperand
#include "vm/UsingHint.h"
#include "gc/ObjectKind-inl.h"
#include "vm/BytecodeIterator-inl.h"
#include "vm/BytecodeLocation-inl.h"
#include "vm/JSObject-inl.h"
using namespace js;
using namespace js::jit;
// Used for building the outermost script.
WarpBuilder::WarpBuilder(WarpSnapshot& snapshot, MIRGenerator& mirGen,
WarpCompilation* warpCompilation)
: WarpBuilderShared(snapshot, mirGen, nullptr),
warpCompilation_(warpCompilation),
graph_(mirGen.graph()),
info_(mirGen.outerInfo()),
scriptSnapshot_(snapshot.rootScript()),
script_(snapshot.rootScript()->script()),
loopStack_(mirGen.alloc()) {
opSnapshotIter_ = scriptSnapshot_->opSnapshots().getFirst();
}
// Used for building inlined scripts.
WarpBuilder::WarpBuilder(WarpBuilder* caller, WarpScriptSnapshot* snapshot,
CompileInfo& compileInfo, CallInfo* inlineCallInfo,
MResumePoint* callerResumePoint)
: WarpBuilderShared(caller->snapshot(), caller->mirGen(), nullptr),
warpCompilation_(caller->warpCompilation()),
graph_(caller->mirGen().graph()),
info_(compileInfo),
scriptSnapshot_(snapshot),
script_(snapshot->script()),
loopStack_(caller->mirGen().alloc()),
callerBuilder_(caller),
callerResumePoint_(callerResumePoint),
inlineCallInfo_(inlineCallInfo) {
opSnapshotIter_ = snapshot->opSnapshots().getFirst();
}
BytecodeSite* WarpBuilder::newBytecodeSite(BytecodeLocation loc) {
jsbytecode* pc = loc.toRawBytecode();
MOZ_ASSERT(info().inlineScriptTree()->script()->containsPC(pc));
return new (alloc()) BytecodeSite(info().inlineScriptTree(), pc);
}
const WarpOpSnapshot* WarpBuilder::getOpSnapshotImpl(
BytecodeLocation loc, WarpOpSnapshot::Kind kind) {
uint32_t offset = loc.bytecodeToOffset(script_);
// Skip snapshots until we get to a snapshot with offset >= offset. This is
// a loop because WarpBuilder can skip unreachable bytecode ops.
while (opSnapshotIter_ && opSnapshotIter_->offset() < offset) {
opSnapshotIter_ = opSnapshotIter_->getNext();
}
if (!opSnapshotIter_ || opSnapshotIter_->offset() != offset ||
opSnapshotIter_->kind() != kind) {
return nullptr;
}
return opSnapshotIter_;
}
void WarpBuilder::initBlock(MBasicBlock* block) {
graph().addBlock(block);
block->setLoopDepth(loopDepth());
current = block;
}
bool WarpBuilder::startNewBlock(MBasicBlock* predecessor, BytecodeLocation loc,
size_t numToPop) {
MBasicBlock* block =
MBasicBlock::NewPopN(graph(), info(), predecessor, newBytecodeSite(loc),
MBasicBlock::NORMAL, numToPop);
if (!block) {
return false;
}
initBlock(block);
return true;
}
bool WarpBuilder::startNewEntryBlock(size_t stackDepth, BytecodeLocation loc) {
MBasicBlock* block =
MBasicBlock::New(graph(), stackDepth, info(), /* maybePred = */ nullptr,
newBytecodeSite(loc), MBasicBlock::NORMAL);
if (!block) {
return false;
}
initBlock(block);
return true;
}
bool WarpBuilder::startNewLoopHeaderBlock(BytecodeLocation loopHead) {
MBasicBlock* header = MBasicBlock::NewPendingLoopHeader(
graph(), info(), current, newBytecodeSite(loopHead));
if (!header) {
return false;
}
initBlock(header);
return loopStack_.emplaceBack(header);
}
bool WarpBuilder::startNewOsrPreHeaderBlock(BytecodeLocation loopHead) {
MOZ_ASSERT(loopHead.is(JSOp::LoopHead));
MOZ_ASSERT(loopHead.toRawBytecode() == info().osrPc());
// Create two blocks:
// * The OSR entry block. This is always the graph's second block and has no
// predecessors. This is the entry point for OSR from the Baseline JIT.
// * The OSR preheader block. This has two predecessors: the OSR entry block
// and the current block.
MBasicBlock* pred = current;
// Create the OSR entry block.
if (!startNewEntryBlock(pred->stackDepth(), loopHead)) {
return false;
}
MBasicBlock* osrBlock = current;
graph().setOsrBlock(osrBlock);
graph().moveBlockAfter(*graph().begin(), osrBlock);
MOsrEntry* entry = MOsrEntry::New(alloc());
osrBlock->add(entry);
// Initialize environment chain.
{
uint32_t slot = info().environmentChainSlot();
MInstruction* envv;
if (usesEnvironmentChain()) {
envv = MOsrEnvironmentChain::New(alloc(), entry);
} else {
// Use an undefined value if the script does not need its environment
// chain, to match the main entry point.
envv = MConstant::NewUndefined(alloc());
}
osrBlock->add(envv);
osrBlock->initSlot(slot, envv);
}
// Initialize return value.
{
MInstruction* returnValue;
if (!script_->noScriptRval()) {
returnValue = MOsrReturnValue::New(alloc(), entry);
} else {
returnValue = MConstant::NewUndefined(alloc());
}
osrBlock->add(returnValue);
osrBlock->initSlot(info().returnValueSlot(), returnValue);
}
// Initialize arguments object.
MInstruction* argsObj = nullptr;
if (info().needsArgsObj()) {
argsObj = MOsrArgumentsObject::New(alloc(), entry);
osrBlock->add(argsObj);
osrBlock->initSlot(info().argsObjSlot(), argsObj);
}
if (info().hasFunMaybeLazy()) {
// Initialize |this| parameter.
MParameter* thisv = MParameter::New(alloc(), MParameter::THIS_SLOT);
osrBlock->add(thisv);
osrBlock->initSlot(info().thisSlot(), thisv);
// Initialize arguments. There are three cases:
//
// 1) There's no ArgumentsObject or it doesn't alias formals. In this case
// we can just use the frame's argument slot.
// 2) The ArgumentsObject aliases formals and the argument is stored in the
// CallObject. Use |undefined| because we can't load from the arguments
// object and code will use the CallObject anyway.
// 3) The ArgumentsObject aliases formals and the argument isn't stored in
// the CallObject. We have to load it from the ArgumentsObject.
for (uint32_t i = 0; i < info().nargs(); i++) {
uint32_t slot = info().argSlotUnchecked(i);
MInstruction* osrv;
if (!info().argsObjAliasesFormals()) {
osrv = MParameter::New(alloc().fallible(), i);
} else if (script_->formalIsAliased(i)) {
osrv = MConstant::New(alloc().fallible(), UndefinedValue());
} else {
osrv = MGetArgumentsObjectArg::New(alloc().fallible(), argsObj, i);
}
if (!osrv) {
return false;
}
current->add(osrv);
current->initSlot(slot, osrv);
}
}
// Initialize locals.
uint32_t nlocals = info().nlocals();
for (uint32_t i = 0; i < nlocals; i++) {
uint32_t slot = info().localSlot(i);
ptrdiff_t offset = BaselineFrame::reverseOffsetOfLocal(i);
MOsrValue* osrv = MOsrValue::New(alloc().fallible(), entry, offset);
if (!osrv) {
return false;
}
current->add(osrv);
current->initSlot(slot, osrv);
}
// Initialize expression stack slots.
uint32_t numStackSlots = current->stackDepth() - info().firstStackSlot();
for (uint32_t i = 0; i < numStackSlots; i++) {
uint32_t slot = info().stackSlot(i);
ptrdiff_t offset = BaselineFrame::reverseOffsetOfLocal(nlocals + i);
MOsrValue* osrv = MOsrValue::New(alloc().fallible(), entry, offset);
if (!osrv) {
return false;
}
current->add(osrv);
current->initSlot(slot, osrv);
}
MStart* start = MStart::New(alloc());
current->add(start);
// Note: phi specialization can add type guard instructions to the OSR entry
// block if needed. See TypeAnalyzer::shouldSpecializeOsrPhis.
// Create the preheader block, with the predecessor block and OSR block as
// predecessors.
if (!startNewBlock(pred, loopHead)) {
return false;
}
pred->end(MGoto::New(alloc(), current));
osrBlock->end(MGoto::New(alloc(), current));
if (!current->addPredecessor(alloc(), osrBlock)) {
return false;
}
return true;
}
bool WarpBuilder::addPendingEdge(BytecodeLocation target, MBasicBlock* block,
uint32_t successor, uint32_t numToPop) {
MOZ_ASSERT(successor < block->lastIns()->numSuccessors());
MOZ_ASSERT(numToPop <= block->stackDepth());
jsbytecode* targetPC = target.toRawBytecode();
PendingEdgesMap::AddPtr p = pendingEdges_.lookupForAdd(targetPC);
if (p) {
return p->value().emplaceBack(block, successor, numToPop);
}
PendingEdges edges;
static_assert(PendingEdges::InlineLength >= 1,
"Appending one element should be infallible");
MOZ_ALWAYS_TRUE(edges.emplaceBack(block, successor, numToPop));
return pendingEdges_.add(p, targetPC, std::move(edges));
}
bool WarpBuilder::build() {
if (script_->isGenerator() || script_->isAsync()) {
resumeAnalysis_.emplace(script_);
if (!resumeAnalysis_->init()) {
return false;
}
if (!resumeAnalysis_->hasResumes()) {
// An async function with no await never suspends, so nothing can resume
// it. Build it like an ordinary function.
resumeAnalysis_.reset();
}
}
if (!buildPrologue()) {
return false;
}
if (!buildBody()) {
return false;
}
MOZ_ASSERT(pendingSuspendSavedSlots_.isNothing(),
"didn't build the AfterYield following a suspend");
// Finish inner-loop blocks build_LoopHead never filled in, because the loop
// is unreachable or only reachable through code Warp doesn't compile, e.g.
// a catch-block. The latter is reachable, so bail out to resume in Baseline
// instead. If this happens frequently, we disable Ion for the script and
// the generator will run in Baseline instead.
for (auto iter = pendingInnerLoopResumes_.iter(); !iter.done(); iter.next()) {
if (!alloc().ensureBallast()) {
return false;
}
MBasicBlock* block = iter.get().value();
MOZ_ASSERT(!block->hasLastIns());
block->add(MBail::New(alloc(), BailoutKind::UncompiledGeneratorResume));
block->end(MUnreachable::New(alloc()));
}
if (!MPhi::markIteratorPhis(*iterators())) {
return false;
}
MOZ_ASSERT_IF(info().osrPc(), graph().osrBlock());
MOZ_ASSERT(loopStack_.empty());
MOZ_ASSERT(loopDepth() == 0);
return true;
}
bool WarpBuilder::buildInline() {
MOZ_ASSERT(!script_->isGenerator() && !script_->isAsync());
if (!buildInlinePrologue()) {
return false;
}
if (!buildBody()) {
return false;
}
MOZ_ASSERT(loopStack_.empty());
return true;
}
MInstruction* WarpBuilder::buildNamedLambdaEnv(MDefinition* callee,
MDefinition* env,
NamedLambdaObject* templateObj,
gc::Heap initialHeap) {
MOZ_ASSERT(templateObj->numDynamicSlots() == 0);
MInstruction* namedLambda =
MNewNamedLambdaObject::New(alloc(), templateObj, initialHeap);
current->add(namedLambda);
// Initialize the object's reserved slots.
size_t enclosingSlot = NamedLambdaObject::enclosingEnvironmentSlot();
size_t lambdaSlot = NamedLambdaObject::lambdaSlot();
auto* storeEnv = MStoreFixedSlot::NewNoPreBarrier(alloc(), namedLambda,
enclosingSlot, env);
auto* storeCallee = MStoreFixedSlot::NewNoPreBarrier(alloc(), namedLambda,
lambdaSlot, callee);
if (initialHeap == gc::Heap::Default) {
// No post barrier is needed here: the object will be allocated in the
// nursery if possible, and if the tenured heap is used instead, a minor
// collection will have been performed that moved env/callee to the tenured
// heap. In debug builds, the AddPostWriteBarriers pass will insert
// MAssertCanElidePostWriteBarrier instructions to check this.
storeEnv->setNeedsPostBarrier(false);
storeCallee->setNeedsPostBarrier(false);
}
current->add(storeEnv);
current->add(storeCallee);
return namedLambda;
}
MInstruction* WarpBuilder::buildCallObject(MDefinition* callee,
MDefinition* env,
CallObject* templateObj,
gc::Heap initialHeap) {
MConstant* templateCst = constant(ObjectValue(*templateObj));
MNewCallObject* callObj =
MNewCallObject::New(alloc(), templateCst, initialHeap);
current->add(callObj);
// Initialize the object's reserved slots.
size_t enclosingSlot = CallObject::enclosingEnvironmentSlot();
size_t calleeSlot = CallObject::calleeSlot();
auto* storeEnv =
MStoreFixedSlot::NewNoPreBarrier(alloc(), callObj, enclosingSlot, env);
auto* storeCallee =
MStoreFixedSlot::NewNoPreBarrier(alloc(), callObj, calleeSlot, callee);
if (initialHeap == gc::Heap::Default) {
// No post barrier is needed here, for the same reason as in
// buildNamedLambdaEnv.
storeEnv->setNeedsPostBarrier(false);
storeCallee->setNeedsPostBarrier(false);
}
current->add(storeEnv);
current->add(storeCallee);
return callObj;
}
bool WarpBuilder::buildEnvironmentChain() {
const WarpEnvironment& env = scriptSnapshot()->environment();
if (env.is<NoEnvironment>()) {
return true;
}
MInstruction* envDef = env.match(
[](const NoEnvironment&) -> MInstruction* {
MOZ_CRASH("Already handled");
},
[this](JSObject* obj) -> MInstruction* {
return constant(ObjectValue(*obj));
},
[this](const FunctionEnvironment& env) -> MInstruction* {
MDefinition* callee = getCallee();
MInstruction* envDef = MFunctionEnvironment::New(alloc(), callee);
current->add(envDef);
if (NamedLambdaObject* obj = env.namedLambdaTemplate) {
envDef = buildNamedLambdaEnv(callee, envDef, obj, env.initialHeap);
}
if (CallObject* obj = env.callObjectTemplate) {
envDef = buildCallObject(callee, envDef, obj, env.initialHeap);
if (!envDef) {
return nullptr;
}
}
return envDef;
});
if (!envDef) {
return false;
}
// Update the environment slot from UndefinedValue only after the initial
// environment is created so that bailout doesn't see a partial environment.
// See: |BaselineStackBuilder::buildBaselineFrame|
current->setEnvironmentChain(envDef);
return true;
}
MDefinition* WarpBuilder::resumeFrameArg(ResumeFrameArgs::Slot slot) {
auto* def = MResumeFrameArg::New(alloc(), slot);
current->add(def);
return def;
}
MDefinition* WarpBuilder::resumeGeneratorObject() {
MDefinition* val = resumeFrameArg(ResumeFrameArgs::GeneratorSlot);
auto* unbox =
MUnbox::New(alloc(), val, MIRType::Object, MUnbox::Mode::Infallible);
current->add(unbox);
return unbox;
}
MDefinition* WarpBuilder::loadGeneratorStackStorage(MDefinition* genObj) {
auto* arrayObj = MLoadFixedSlotAndUnbox::New(
alloc(), genObj, AbstractGeneratorObject::stackStorageSlot(),
MUnbox::Mode::Infallible, MIRType::Object);
current->add(arrayObj);
return arrayObj;
}
bool WarpBuilder::startResumePath(MBasicBlock* from, BytecodeLocation loc,
MBasicBlock** normalBlock) {
auto* isResuming = MIsResumingGenerator::New(alloc());
from->add(isResuming);
if (!startNewBlock(from, loc)) {
return false;
}
*normalBlock = current;
if (!startNewBlock(from, loc)) {
return false;
}
from->end(MTest::New(alloc(), isResuming, /* ifTrue = */ current,
/* ifFalse = */ *normalBlock));
return true;
}
bool WarpBuilder::linkDispatchEntry(MBasicBlock* from, size_t successorIndex,
const DispatchEntry& entry,
BytecodeLocation loc) {
MOZ_ASSERT(successorIndex < from->lastIns()->numSuccessors());
if (entry.isAfterYield()) {
// We can jump directly to the AfterYield op.
BytecodeLocation afterYield =
resumeAnalysis_->afterYieldLocationAt(entry.resumeIndex());
if (!addPendingEdge(afterYield, from, successorIndex)) {
return false;
}
} else {
// We have a loop that contains an AfterYield. Create a new block to jump
// to the loop header. See buildLoopResumeMerge.
if (!startNewBlock(from, loc)) {
return false;
}
from->lastIns()->initSuccessor(successorIndex, current);
PendingInnerLoopResumesMap::AddPtr p =
pendingInnerLoopResumes_.lookupForAdd(entry.innerLoopHeadOffset());
MOZ_ASSERT(!p, "each loop is targeted by a single dispatch entry");
if (!pendingInnerLoopResumes_.add(p, entry.innerLoopHeadOffset(),
current)) {
return false;
}
}
setTerminatedBlock();
return true;
}
MBasicBlock* WarpBuilder::takePendingInnerLoopResume(
BytecodeLocation loopHead) {
MOZ_ASSERT(loopHead.is(JSOp::LoopHead));
uint32_t loopHeadOffset = loopHead.bytecodeToOffset(script_);
auto p = pendingInnerLoopResumes_.lookup(loopHeadOffset);
MOZ_RELEASE_ASSERT(p);
MBasicBlock* block = p->value();
pendingInnerLoopResumes_.remove(p);
return block;
}
bool WarpBuilder::buildResumeIndexDispatch(BytecodeLocation loc,
DispatchEntrySpan entries) {
MOZ_ASSERT(!entries.empty());
// If there's one possible target, we don't need to check the resume index.
// Note that MTableSwitch::foldsTo can't do this for us due to the |default|
// successor.
if (entries.size() == 1) {
current->end(MGoto::New(alloc(), nullptr));
return linkDispatchEntry(current, MGoto::TargetIndex, entries[0], loc);
}
auto* resumeIndexVal = resumeFrameArg(ResumeFrameArgs::ResumeIndexSlot);
auto* resumeIndex = MUnbox::New(alloc(), resumeIndexVal, MIRType::Int32,
MUnbox::Mode::Infallible);
current->add(resumeIndex);
// If there are two possible targets, we can use a compare-and-branch. The
// cases are ordered and the index must be in one of them, so compare against
// the first resume index in the second entry's range.
if (entries.size() == 2) {
int32_t second = int32_t(entries[1].resumeIndices().begin);
MOZ_ASSERT(int32_t(entries[0].resumeIndices().last()) < second);
auto* isFirst =
MCompare::New(alloc(), resumeIndex, constant(Int32Value(second)),
JSOp::Lt, MCompare::Compare_Int32);
current->add(isFirst);
current->end(MTest::New(alloc(), isFirst, /* ifTrue = */ nullptr,
/* ifFalse = */ nullptr));
MBasicBlock* from = current;
return linkDispatchEntry(from, MTest::TrueBranchIndex, entries[0], loc) &&
linkDispatchEntry(from, MTest::FalseBranchIndex, entries[1], loc);
}
// Create a table switch that covers all resume indices.
int32_t low = int32_t(entries[0].resumeIndices().begin);
int32_t high = int32_t(entries[entries.size() - 1].resumeIndices().last());
auto* tableswitch = MTableSwitch::New(alloc(), resumeIndex, low, high);
current->end(tableswitch);
// Out-of-range indices are unreachable: a suspended generator's resume index
// is always in |entries|. We create a |default| block with MUnreachable for
// these indices.
size_t defaultIndex;
if (!tableswitch->addDefault(nullptr, &defaultIndex)) {
return false;
}
// Add a single successor for each entry (either an AfterYield op we can jump
// to or an inner loop with AfterYields) and a case for each of the entry's
// resume indices.
MBasicBlock* from = current;
mozilla::DebugOnly<int32_t> nextResumeIndex = low;
for (const DispatchEntry& entry : entries) {
size_t successorIndex;
if (!tableswitch->addSuccessor(nullptr, &successorIndex)) {
return false;
}
ResumeIndexRange range = entry.resumeIndices();
for (uint32_t index = range.begin; index < range.end; index++) {
MOZ_ASSERT(int32_t(index) == nextResumeIndex);
if (!tableswitch->addCase(successorIndex)) {
return false;
}
nextResumeIndex++;
}
if (!linkDispatchEntry(from, successorIndex, entry, loc)) {
return false;
}
}
MOZ_ASSERT(nextResumeIndex == high + 1);
// Create the |default| block with MUnreachable.
if (!startNewBlock(from, loc)) {
return false;
}
tableswitch->initSuccessor(defaultIndex, current);
current->end(MUnreachable::New(alloc()));
setTerminatedBlock();
return true;
}
bool WarpBuilder::buildLoopResumeMerge(BytecodeLocation loopHead) {
MBasicBlock* preheader = current;
// Create a new block that merges the normal path and the resume path. This
// ensures the loop header has a single forward predecessor.
if (!startNewBlock(preheader, loopHead)) {
return false;
}
MBasicBlock* merge = current;
// Finish the resume block created in linkDispatchEntry now that we know the
// stack depth.
MBasicBlock* innerLoopResume = takePendingInnerLoopResume(loopHead);
MOZ_ASSERT(!innerLoopResume->hasLastIns());
current = innerLoopResume;
// Push |undefined| for the header's expression stack instead of the saved
// values: the JSOp::AfterYield op will rebuild the expression stack from the
// generator object, and a bailout on the resume path restores the values in
// Baseline's prologue, so these are never read.
//
// Restoring these stack slots here isn't an option due to bytecode generated
// for yield* loops. The loop header has [next, iter, received, resumeKind]
// slots but the suspend saves only [next, iter].
MOZ_ASSERT(innerLoopResume->stackDepth() <= merge->stackDepth());
if (innerLoopResume->stackDepth() < merge->stackDepth()) {
MConstant* undef = constant(UndefinedValue());
while (current->stackDepth() < merge->stackDepth()) {
current->push(undef);
}
}
innerLoopResume->end(MGoto::New(alloc(), merge));
if (!merge->addPredecessor(alloc(), innerLoopResume)) {
return false;
}
preheader->end(MGoto::New(alloc(), merge));
current = merge;
return true;
}
bool WarpBuilder::buildPrologueResumeDispatch(BytecodeLocation startLoc) {
MBasicBlock* entryBlock = current;
MBasicBlock* normalBlock = nullptr;
if (!startResumePath(entryBlock, startLoc, &normalBlock)) {
return false;
}
// Guard against over-recursion. The resume path must not handle interrupts
// because interrupt callbacks can run JS/Debugger code, and this frame's pc
// is still the script start with its locals and expression stack not yet
// restored.
current->add(
MCheckOverRecursed::New(alloc(), /* isResumingGenerator = */ true));
// Restore the environment chain.
MDefinition* genObj = resumeGeneratorObject();
auto* envChain = MLoadFixedSlotAndUnbox::New(
alloc(), genObj, AbstractGeneratorObject::envChainSlot(),
MUnbox::Mode::Infallible, MIRType::Object);
current->add(envChain);
current->setEnvironmentChain(envChain);
// Restore the locals. The expression stack slots are restored at the
// AfterYield op.
uint32_t nfixed = info().nlocals();
if (nfixed > 0) {
MDefinition* arrayObj = loadGeneratorStackStorage(genObj);
auto* elements = MElements::New(alloc(), arrayObj);
current->add(elements);
for (uint32_t i = 0; i < nfixed; i++) {
if (!alloc().ensureBallast()) {
return false;
}
auto* load = MLoadElement::New(alloc(), elements, constant(Int32Value(i)),
/* needsHoleCheck = */ false);
current->add(load);
current->setSlot(info().localSlot(i), load);
}
}
// Restore the arguments object.
if (info().needsArgsObj()) {
auto* argsObj = MLoadFixedSlotAndUnbox::New(
alloc(), genObj, AbstractGeneratorObject::argsObjectSlot(),
MUnbox::Mode::Infallible, MIRType::Object);
current->add(argsObj);
current->setSlot(info().argsObjSlot(), argsObj);
}
// Build the resume index dispatch.
DispatchEntrySpan entries = resumeAnalysis_->prologueDispatchEntries();
MOZ_ASSERT(!entries.empty(), "a script with no suspend has no dispatch");
if (!buildResumeIndexDispatch(startLoc, entries)) {
return false;
}
current = normalBlock;
return true;
}
bool WarpBuilder::buildPrologue() {
BytecodeLocation startLoc(script_, script_->code());
if (!startNewEntryBlock(info().firstStackSlot(), startLoc)) {
return false;
}
if (info().hasFunMaybeLazy()) {
// Initialize |this|.
MParameter* param = MParameter::New(alloc(), MParameter::THIS_SLOT);
current->add(param);
current->initSlot(info().thisSlot(), param);
// Initialize arguments.
for (uint32_t i = 0; i < info().nargs(); i++) {
MParameter* param = MParameter::New(alloc().fallible(), i);
if (!param) {
return false;
}
current->add(param);
current->initSlot(info().argSlotUnchecked(i), param);
}
}
MConstant* undef = constant(UndefinedValue());
// Initialize local slots.
for (uint32_t i = 0; i < info().nlocals(); i++) {
current->initSlot(info().localSlot(i), undef);
}
// Initialize the environment chain, return value, and arguments object slots.
current->initSlot(info().environmentChainSlot(), undef);
current->initSlot(info().returnValueSlot(), undef);
if (info().needsArgsObj()) {
current->initSlot(info().argsObjSlot(), undef);
}
current->add(MStart::New(alloc()));
// Build the prologue's resume dispatch if this is a generator or async
// function. After this, |current| is the normal non-resume path.
if (resumeAnalysis_.isSome()) {
if (!buildPrologueResumeDispatch(startLoc)) {
return false;
}
}
// Guard against over-recursion.
auto* check =
MCheckOverRecursed::New(alloc(), /* isResumingGenerator = */ false);
current->add(check);
if (!buildEnvironmentChain()) {
return false;
}
#ifdef JS_CACHEIR_SPEW
if (snapshot().needsFinalWarmUpCount()) {
MIncrementWarmUpCounter* ins =
MIncrementWarmUpCounter::New(alloc(), script_);
current->add(ins);
}
#endif
return true;
}
bool WarpBuilder::buildInlinePrologue() {
// Generate entry block.
BytecodeLocation startLoc(script_, script_->code());
if (!startNewEntryBlock(info().firstStackSlot(), startLoc)) {
return false;
}
current->setCallerResumePoint(callerResumePoint());
// Connect the entry block to the last block in the caller's graph.
MBasicBlock* pred = callerBuilder()->current;
MOZ_ASSERT(pred == callerResumePoint()->block());
pred->end(MGoto::New(alloc(), current));
if (!current->addPredecessorWithoutPhis(pred)) {
return false;
}
MConstant* undef = constant(UndefinedValue());
// Initialize env chain slot to Undefined. It's set later by
// |buildEnvironmentChain|.
current->initSlot(info().environmentChainSlot(), undef);
// Initialize |return value| slot.
current->initSlot(info().returnValueSlot(), undef);
// Initialize |arguments| slot if needed.
if (info().needsArgsObj()) {
current->initSlot(info().argsObjSlot(), undef);
}
// Initialize |this| slot.
current->initSlot(info().thisSlot(), inlineCallInfo()->thisArg());
uint32_t callerArgs = inlineCallInfo()->argc();
uint32_t actualArgs = info().nargs();
uint32_t passedArgs = std::min<uint32_t>(callerArgs, actualArgs);
// Initialize actually set arguments.
for (uint32_t i = 0; i < passedArgs; i++) {
MDefinition* arg = inlineCallInfo()->getArg(i);
current->initSlot(info().argSlotUnchecked(i), arg);
}
// Pass undefined for missing arguments.
for (uint32_t i = passedArgs; i < actualArgs; i++) {
current->initSlot(info().argSlotUnchecked(i), undef);
}
// Initialize local slots.
for (uint32_t i = 0; i < info().nlocals(); i++) {
current->initSlot(info().localSlot(i), undef);
}
MOZ_ASSERT(current->entryResumePoint()->stackDepth() == info().totalSlots());
if (!buildEnvironmentChain()) {
return false;
}
return true;
}
#ifdef DEBUG
// In debug builds, after compiling a bytecode op, this class is used to check
// that all values popped by this opcode either:
//
// (1) Have the ImplicitlyUsed flag set on them.
// (2) Have more uses than before compiling this op (the value is
// used as operand of a new MIR instruction).
//
// This is used to catch problems where WarpBuilder pops a value without
// adding any SSA uses and doesn't call setImplicitlyUsedUnchecked on it.
class MOZ_RAII WarpPoppedValueUseChecker {
Vector<MDefinition*, 4, SystemAllocPolicy> popped_;
Vector<size_t, 4, SystemAllocPolicy> poppedUses_;
MBasicBlock* current_;
BytecodeLocation loc_;
public:
WarpPoppedValueUseChecker(MBasicBlock* current, BytecodeLocation loc)
: current_(current), loc_(loc) {}
[[nodiscard]] bool init() {
// Don't require SSA uses for values popped by these ops.
switch (loc_.getOp()) {
case JSOp::Pop:
case JSOp::PopN:
case JSOp::DupAt:
case JSOp::Dup:
case JSOp::Dup2:
case JSOp::Pick:
case JSOp::Unpick:
case JSOp::Swap:
case JSOp::SetArg:
case JSOp::SetLocal:
case JSOp::InitLexical:
case JSOp::SetRval:
case JSOp::Void:
// Basic stack/local/argument management opcodes.
return true;
case JSOp::Case:
case JSOp::Default:
// These ops have to pop the switch value when branching but don't
// actually use it.
return true;
default:
break;
}
unsigned nuses = loc_.useCount();
for (unsigned i = 0; i < nuses; i++) {
MDefinition* def = current_->peek(-int32_t(i + 1));
if (!popped_.append(def) || !poppedUses_.append(def->defUseCount())) {
return false;
}
}
return true;
}
void checkAfterOp() {
for (size_t i = 0; i < popped_.length(); i++) {
// First value popped by JSOp::EndIter is not used at all, it's similar
// to JSOp::Pop above.
if (loc_.is(JSOp::EndIter) && i == 0) {
continue;
}
MOZ_ASSERT(popped_[i]->isImplicitlyUsed() ||
popped_[i]->defUseCount() > poppedUses_[i]);
}
}
};
#endif
bool WarpBuilder::buildBody() {
for (BytecodeLocation loc : AllBytecodesIterable(script_)) {
if (mirGen().shouldCancel("WarpBuilder (opcode loop)")) {
return false;
}
// Skip unreachable ops (for example code after a 'return' or 'throw') until
// we get to the next jump target.
if (hasTerminatedBlock()) {
// Finish any "broken" loops with an unreachable backedge. For example:
//
// do {
// ...
// return;
// ...
// } while (x);
//
// This loop never actually loops.
if (loc.isBackedge() && !loopStack_.empty()) {
BytecodeLocation loopHead(script_,
loopStack_.back().header()->entryPC());
if (loc.isBackedgeForLoophead(loopHead)) {
decLoopDepth();
loopStack_.popBack();
}
}
if (!loc.isJumpTarget()) {
continue;
}
}
if (!alloc().ensureBallast()) {
return false;
}
#ifdef DEBUG
WarpPoppedValueUseChecker useChecker(current, loc);
if (!useChecker.init()) {
return false;
}
#endif
bool wantPreciseLineNumbers =
js::jit::PerfEnabled() || mirGen().isProfilerInstrumentationEnabled();
if (wantPreciseLineNumbers && !hasTerminatedBlock()) {
current->updateTrackedSite(newBytecodeSite(loc));
}
JSOp op = loc.getOp();
#define BUILD_OP(OP, ...) \
case JSOp::OP: \
if (MOZ_UNLIKELY(!this->build_##OP(loc))) { \
return false; \
} \
break;
switch (op) { FOR_EACH_OPCODE(BUILD_OP) }
#undef BUILD_OP
#ifdef DEBUG
useChecker.checkAfterOp();
#endif
}
return true;
}
#define DEF_OP(OP) \
bool WarpBuilder::build_##OP(BytecodeLocation) { \
MOZ_CRASH("Unsupported op"); \
}
WARP_UNSUPPORTED_OPCODE_LIST(DEF_OP)
#undef DEF_OP
bool WarpBuilder::build_Nop(BytecodeLocation) { return true; }
bool WarpBuilder::build_NopDestructuring(BytecodeLocation) { return true; }
bool WarpBuilder::build_NopIsAssignOp(BytecodeLocation) { return true; }
bool WarpBuilder::build_TryDestructuring(BytecodeLocation) {
// Set the hasTryBlock flag to turn off optimizations that eliminate dead
// resume points operands because the exception handler code for
// TryNoteKind::Destructuring is effectively a (specialized) catch-block.
graph().setHasTryBlock();
return true;
}
bool WarpBuilder::build_Lineno(BytecodeLocation) { return true; }
bool WarpBuilder::build_DebugLeaveLexicalEnv(BytecodeLocation) { return true; }
bool WarpBuilder::build_Undefined(BytecodeLocation) {
pushConstant(UndefinedValue());
return true;
}
bool WarpBuilder::build_Void(BytecodeLocation) {
current->pop();
pushConstant(UndefinedValue());
return true;
}
bool WarpBuilder::build_Null(BytecodeLocation) {
pushConstant(NullValue());
return true;
}
bool WarpBuilder::build_Hole(BytecodeLocation) {
pushConstant(MagicValue(JS_ELEMENTS_HOLE));
return true;
}
bool WarpBuilder::build_Uninitialized(BytecodeLocation) {
pushConstant(MagicValue(JS_UNINITIALIZED_LEXICAL));
return true;
}
bool WarpBuilder::build_IsConstructing(BytecodeLocation) {
pushConstant(MagicValue(JS_IS_CONSTRUCTING));
return true;
}
bool WarpBuilder::build_False(BytecodeLocation) {
pushConstant(BooleanValue(false));
return true;
}
bool WarpBuilder::build_True(BytecodeLocation) {
pushConstant(BooleanValue(true));
return true;
}
bool WarpBuilder::build_Pop(BytecodeLocation) {
current->pop();
return true;
}
bool WarpBuilder::build_PopN(BytecodeLocation loc) {
for (uint32_t i = 0, n = loc.getPopCount(); i < n; i++) {
current->pop();
}
return true;
}
bool WarpBuilder::build_Dup(BytecodeLocation) {
current->pushSlot(current->stackDepth() - 1);
return true;
}
bool WarpBuilder::build_Dup2(BytecodeLocation) {
uint32_t lhsSlot = current->stackDepth() - 2;
uint32_t rhsSlot = current->stackDepth() - 1;
current->pushSlot(lhsSlot);
current->pushSlot(rhsSlot);
return true;
}
bool WarpBuilder::build_DupAt(BytecodeLocation loc) {
current->pushSlot(current->stackDepth() - 1 - loc.getDupAtIndex());
return true;
}
bool WarpBuilder::build_Swap(BytecodeLocation) {
current->swapAt(-1);
return true;
}
bool WarpBuilder::build_Pick(BytecodeLocation loc) {
int32_t depth = -int32_t(loc.getPickDepth());
current->pick(depth);
return true;
}
bool WarpBuilder::build_Unpick(BytecodeLocation loc) {
int32_t depth = -int32_t(loc.getUnpickDepth());
current->unpick(depth);
return true;
}
bool WarpBuilder::build_Zero(BytecodeLocation) {
pushConstant(Int32Value(0));
return true;
}
bool WarpBuilder::build_One(BytecodeLocation) {
pushConstant(Int32Value(1));
return true;
}
bool WarpBuilder::build_Int8(BytecodeLocation loc) {
pushConstant(Int32Value(loc.getInt8()));
return true;
}
bool WarpBuilder::build_Uint16(BytecodeLocation loc) {
pushConstant(Int32Value(loc.getUint16()));
return true;
}
bool WarpBuilder::build_Uint24(BytecodeLocation loc) {
pushConstant(Int32Value(loc.getUint24()));
return true;
}
bool WarpBuilder::build_Int32(BytecodeLocation loc) {
pushConstant(Int32Value(loc.getInt32()));
return true;
}
bool WarpBuilder::build_Double(BytecodeLocation loc) {
pushConstant(loc.getInlineValue());
return true;
}
bool WarpBuilder::build_BigInt(BytecodeLocation loc) {
BigInt* bi = loc.getBigInt(script_);
pushConstant(BigIntValue(bi));
return true;
}
bool WarpBuilder::build_String(BytecodeLocation loc) {
JSString* str = loc.getString(script_);
pushConstant(StringValue(str));
return true;
}
bool WarpBuilder::build_Symbol(BytecodeLocation loc) {
uint32_t which = loc.getSymbolIndex();
JS::Symbol* sym = mirGen().runtime->wellKnownSymbols().get(which);
pushConstant(SymbolValue(sym));
return true;
}
bool WarpBuilder::build_RegExp(BytecodeLocation loc) {
RegExpObject* reObj = loc.getRegExp(script_);
auto* snapshot = getOpSnapshot<WarpRegExp>(loc);
MRegExp* regexp = MRegExp::New(alloc(), reObj, snapshot->hasShared());
current->add(regexp);
current->push(regexp);
return true;
}
bool WarpBuilder::build_Return(BytecodeLocation) {
MDefinition* def = current->pop();
MReturn* ret = MReturn::New(alloc(), def);
current->end(ret);
if (!graph().addReturn(current)) {
return false;
}
setTerminatedBlock();
return true;
}
bool WarpBuilder::build_RetRval(BytecodeLocation) {
MDefinition* rval;
if (script_->noScriptRval()) {
rval = constant(UndefinedValue());
} else {
rval = current->getSlot(info().returnValueSlot());
}
MReturn* ret = MReturn::New(alloc(), rval);
current->end(ret);
if (!graph().addReturn(current)) {
return false;
}
setTerminatedBlock();
return true;
}
bool WarpBuilder::build_SetRval(BytecodeLocation) {
MOZ_ASSERT(!script_->noScriptRval());
MDefinition* rval = current->pop();
current->setSlot(info().returnValueSlot(), rval);
return true;
}
bool WarpBuilder::build_GetRval(BytecodeLocation) {
MOZ_ASSERT(!script_->noScriptRval());
MDefinition* rval = current->getSlot(info().returnValueSlot());
current->push(rval);
return true;
}
bool WarpBuilder::build_GetLocal(BytecodeLocation loc) {
current->pushLocal(loc.local());
return true;
}
bool WarpBuilder::build_SetLocal(BytecodeLocation loc) {
current->setLocal(loc.local());
return true;
}
bool WarpBuilder::build_InitLexical(BytecodeLocation loc) {
current->setLocal(loc.local());
return true;
}
bool WarpBuilder::build_GetArg(BytecodeLocation loc) {
uint32_t arg = loc.arg();
if (info().argsObjAliasesFormals()) {
MDefinition* argsObj = current->argumentsObject();
auto* getArg = MGetArgumentsObjectArg::New(alloc(), argsObj, arg);
current->add(getArg);
current->push(getArg);
} else {
current->pushArg(arg);
}
return true;
}
bool WarpBuilder::build_GetFrameArg(BytecodeLocation loc) {
current->pushArgUnchecked(loc.arg());
return true;
}
bool WarpBuilder::build_SetArg(BytecodeLocation loc) {
uint32_t arg = loc.arg();
MDefinition* val = current->peek(-1);
if (!info().argsObjAliasesFormals()) {
// Either |arguments| is never referenced within this function, or
// it doesn't map to the actual arguments values. Either way, we
// don't need to worry about synchronizing the argument values
// when writing to them.
//
// If this SetArg is pruned, a bailout may resume before it, in which case
// Baseline will store `val` into the formal. If formals are observable via
// Function.arguments, `val` should not be optimized out.
if (info().isObservableSlot(info().argSlot(arg))) {
val->setImplicitlyUsedUnchecked();
}
current->setArg(arg);
return true;
}
// If an arguments object is in use, and it aliases formals, then all SetArgs
// must go through the arguments object.
MDefinition* argsObj = current->argumentsObject();
auto* ins = MSetArgumentsObjectArg::New(alloc(), argsObj, val, arg);
current->add(ins);
return resumeAfter(ins, loc);
}
bool WarpBuilder::build_ArgumentsLength(BytecodeLocation) {
if (inlineCallInfo()) {
pushConstant(Int32Value(inlineCallInfo()->argc()));
} else {
auto* argsLength = MArgumentsLength::New(alloc());
current->add(argsLength);
current->push(argsLength);
}
return true;
}
bool WarpBuilder::build_GetActualArg(BytecodeLocation) {
MDefinition* index = current->pop();
MInstruction* arg;
if (inlineCallInfo()) {
arg = MGetInlinedArgument::New(alloc(), index, *inlineCallInfo());
if (!arg) {
return false;
}
} else {
arg = MGetFrameArgument::New(alloc(), index);
}
current->add(arg);
current->push(arg);
return true;
}
bool WarpBuilder::build_ToNumeric(BytecodeLocation loc) {
return buildUnaryOp(loc);
}
bool WarpBuilder::buildUnaryOp(BytecodeLocation loc) {
MDefinition* value = current->pop();
return buildIC(loc, CacheKind::UnaryArith, {value});
}
bool WarpBuilder::build_Inc(BytecodeLocation loc) { return buildUnaryOp(loc); }
bool WarpBuilder::build_Dec(BytecodeLocation loc) { return buildUnaryOp(loc); }
bool WarpBuilder::build_Pos(BytecodeLocation loc) { return buildUnaryOp(loc); }
bool WarpBuilder::build_Neg(BytecodeLocation loc) { return buildUnaryOp(loc); }
bool WarpBuilder::build_BitNot(BytecodeLocation loc) {
return buildUnaryOp(loc);
}
bool WarpBuilder::buildBinaryOp(BytecodeLocation loc) {
MDefinition* right = current->pop();
MDefinition* left = current->pop();
return buildIC(loc, CacheKind::BinaryArith, {left, right});
}
bool WarpBuilder::build_Add(BytecodeLocation loc) { return buildBinaryOp(loc); }
bool WarpBuilder::build_Sub(BytecodeLocation loc) { return buildBinaryOp(loc); }
bool WarpBuilder::build_Mul(BytecodeLocation loc) { return buildBinaryOp(loc); }
bool WarpBuilder::build_Div(BytecodeLocation loc) { return buildBinaryOp(loc); }
bool WarpBuilder::build_Mod(BytecodeLocation loc) { return buildBinaryOp(loc); }
bool WarpBuilder::build_Pow(BytecodeLocation loc) { return buildBinaryOp(loc); }
bool WarpBuilder::build_BitAnd(BytecodeLocation loc) {
return buildBinaryOp(loc);
}
bool WarpBuilder::build_BitOr(BytecodeLocation loc) {
return buildBinaryOp(loc);
}
bool WarpBuilder::build_BitXor(BytecodeLocation loc) {
return buildBinaryOp(loc);
}
bool WarpBuilder::build_Lsh(BytecodeLocation loc) { return buildBinaryOp(loc); }
bool WarpBuilder::build_Rsh(BytecodeLocation loc) { return buildBinaryOp(loc); }
bool WarpBuilder::build_Ursh(BytecodeLocation loc) {
return buildBinaryOp(loc);
}
bool WarpBuilder::buildCompareOp(BytecodeLocation loc) {
MDefinition* right = current->pop();
MDefinition* left = current->pop();
return buildIC(loc, CacheKind::Compare, {left, right});
}
bool WarpBuilder::build_Eq(BytecodeLocation loc) { return buildCompareOp(loc); }
bool WarpBuilder::build_Ne(BytecodeLocation loc) { return buildCompareOp(loc); }
bool WarpBuilder::build_Lt(BytecodeLocation loc) { return buildCompareOp(loc); }
bool WarpBuilder::build_Le(BytecodeLocation loc) { return buildCompareOp(loc); }
bool WarpBuilder::build_Gt(BytecodeLocation loc) { return buildCompareOp(loc); }
bool WarpBuilder::build_Ge(BytecodeLocation loc) { return buildCompareOp(loc); }
bool WarpBuilder::build_StrictEq(BytecodeLocation loc) {
return buildCompareOp(loc);
}
bool WarpBuilder::build_StrictNe(BytecodeLocation loc) {
return buildCompareOp(loc);
}
bool WarpBuilder::buildStrictConstantEqOp(BytecodeLocation loc,
JSOp compareOp) {
auto operand = loc.getConstantCompareOperand();
MDefinition* value = current->pop();
switch (operand.type()) {
case ConstantCompareOperand::EncodedType::Int32: {
if (value->type() == MIRType::Int32) {
MConstant* constant = MConstant::NewInt32(alloc(), operand.toInt32());
current->add(constant);
auto* compare = MCompare::New(alloc(), value, constant, compareOp,
MCompare::Compare_Int32);
current->add(compare);
current->push(compare);
return true;
}
auto* ins = MStrictConstantCompareInt32::New(
alloc(), value, operand.toInt32(), compareOp);
current->add(ins);
current->push(ins);
return true;
}
case ConstantCompareOperand::EncodedType::Boolean: {
if (value->type() == MIRType::Boolean) {
MConstant* constant = MConstant::NewInt32(alloc(), operand.toBoolean());
current->add(constant);
auto* toBoolToInt32 = MBooleanToInt32::New(alloc(), value);
current->add(toBoolToInt32);
auto* compare = MCompare::New(alloc(), toBoolToInt32, constant,
compareOp, MCompare::Compare_Int32);
current->add(compare);
current->push(compare);
return true;
}
auto* ins = MStrictConstantCompareBoolean::New(
alloc(), value, operand.toBoolean(), compareOp);
current->add(ins);
current->push(ins);
return true;
}
case ConstantCompareOperand::EncodedType::Null: {
MConstant* constant = MConstant::NewNull(alloc());
current->add(constant);
auto* ins = MCompare::New(alloc(), value, constant, compareOp,
MCompare::Compare_Null);
current->add(ins);
current->push(ins);
return true;
}
case ConstantCompareOperand::EncodedType::Undefined: {
MConstant* constant = MConstant::NewUndefined(alloc());
current->add(constant);
auto* ins = MCompare::New(alloc(), value, constant, compareOp,
MCompare::Compare_Undefined);
current->add(ins);
current->push(ins);
return true;
}
}
return true;
}
bool WarpBuilder::build_StrictConstantEq(BytecodeLocation loc) {
return buildStrictConstantEqOp(loc, JSOp::StrictEq);
}
bool WarpBuilder::build_StrictConstantNe(BytecodeLocation loc) {
return buildStrictConstantEqOp(loc, JSOp::StrictNe);
}
bool WarpBuilder::build_AddDisposable(BytecodeLocation loc) {
MOZ_ASSERT(usesEnvironmentChain());
UsingHint hint = loc.getUsingHint();
MDefinition* needsClosure = current->pop();
MDefinition* method = current->pop();
MDefinition* obj = current->pop();
MDefinition* env = current->environmentChain();
MAddDisposableResource* ins = MAddDisposableResource::New(
alloc(), env, obj, method, needsClosure, uint8_t(hint));
current->add(ins);
return resumeAfter(ins, loc);
}
bool WarpBuilder::build_TakeDisposeCapability(BytecodeLocation loc) {
MOZ_ASSERT(usesEnvironmentChain());
MDefinition* env = current->environmentChain();
MTakeDisposeCapability* ins = MTakeDisposeCapability::New(alloc(), env);
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
// Returns true iff the MTest added for |op| has a true-target corresponding
// with the join point in the bytecode.
static bool TestTrueTargetIsJoinPoint(JSOp op) {
switch (op) {
case JSOp::JumpIfTrue:
case JSOp::Or:
case JSOp::Case:
return true;
case JSOp::JumpIfFalse:
case JSOp::And:
case JSOp::Coalesce:
return false;
default:
MOZ_CRASH("Unexpected op");
}
}
bool WarpBuilder::build_JumpTarget(BytecodeLocation loc) {
PendingEdgesMap::Ptr p = pendingEdges_.lookup(loc.toRawBytecode());
if (!p) {
// No (reachable) jumps so this is just a no-op.
return true;
}
PendingEdges edges(std::move(p->value()));
pendingEdges_.remove(p);
MOZ_ASSERT(!edges.empty());
// Create join block if there's fall-through from the previous bytecode op.
if (!hasTerminatedBlock()) {
MBasicBlock* pred = current;
if (!startNewBlock(pred, loc)) {
return false;
}
pred->end(MGoto::New(alloc(), current));
}
for (const PendingEdge& edge : edges) {
MBasicBlock* source = edge.block();
uint32_t numToPop = edge.numToPop();
if (hasTerminatedBlock()) {
if (!startNewBlock(source, loc, numToPop)) {
return false;
}
} else {
MOZ_ASSERT(source->stackDepth() - numToPop == current->stackDepth());
if (!current->addPredecessorPopN(alloc(), source, numToPop)) {
return false;
}
}
MOZ_ASSERT(source->lastIns()->isTest() || source->lastIns()->isGoto() ||
source->lastIns()->isTableSwitch());
source->lastIns()->initSuccessor(edge.successor(), current);
}
MOZ_ASSERT(!hasTerminatedBlock());
return true;
}
bool WarpBuilder::addIteratorLoopPhis(BytecodeLocation loopHead) {
// When unwinding the stack for a thrown exception, the exception handler must
// close live iterators. For ForIn and Destructuring loops, the exception
// handler needs access to values on the stack. To prevent them from being
// optimized away (and replaced with the JS_OPTIMIZED_OUT MagicValue), we need
// to mark the phis (and phis they flow into) as having implicit uses.
// See ProcessTryNotes in vm/Interpreter.cpp and CloseLiveIteratorIon in
// jit/JitFrames.cpp
MOZ_ASSERT(current->stackDepth() >= info().firstStackSlot());
bool emptyStack = current->stackDepth() == info().firstStackSlot();
if (emptyStack) {
return true;
}
jsbytecode* loopHeadPC = loopHead.toRawBytecode();
for (TryNoteIterAllNoGC tni(script_, loopHeadPC); !tni.done(); ++tni) {
const TryNote& tn = **tni;
// Stop if we reach an outer loop because outer loops were already
// processed when we visited their loop headers.
if (tn.isLoop()) {
BytecodeLocation tnStart = script_->offsetToLocation(tn.start);
if (tnStart != loopHead) {
MOZ_ASSERT(tnStart.is(JSOp::LoopHead));
MOZ_ASSERT(tnStart < loopHead);
return true;
}
}
switch (tn.kind()) {
case TryNoteKind::Destructuring:
case TryNoteKind::ForIn: {
// For for-in loops we add the iterator object to iterators(). For
// destructuring loops we add the "done" value that's on top of the
// stack and used in the exception handler.
MOZ_ASSERT(tn.stackDepth >= 1);
uint32_t slot = info().stackSlot(tn.stackDepth - 1);
MPhi* phi = current->getSlot(slot)->toPhi();
if (!iterators()->append(phi)) {
return false;
}
break;
}
case TryNoteKind::Loop:
case TryNoteKind::ForOf:
// Regular loops do not have iterators to close. ForOf loops handle
// unwinding using catch blocks.
break;
default:
break;
}
}
return true;
}
bool WarpBuilder::build_LoopHead(BytecodeLocation loc) {
// All loops have the following bytecode structure:
//
// LoopHead
// ...
// JumpIfTrue/Goto to LoopHead
if (hasTerminatedBlock()) {
// The whole loop is unreachable.
return true;
}
// If this loop contains a yield or await, the generator resume dispatch has
// to descend into this header to avoid irreducible control flow. Create a new
// block to merge the normal and resume paths before the loop header.
DispatchEntrySpan entries;
if (resumeAnalysis_.isSome()) {
entries = resumeAnalysis_->loopDispatchEntries(loc);
if (!entries.empty() && !buildLoopResumeMerge(loc)) {
return false;
}
}
// Handle OSR from Baseline JIT code.
if (loc.toRawBytecode() == info().osrPc()) {
if (!startNewOsrPreHeaderBlock(loc)) {
return false;
}
}
incLoopDepth();
MBasicBlock* pred = current;
if (!startNewLoopHeaderBlock(loc)) {
return false;
}
pred->end(MGoto::New(alloc(), current));
if (!addIteratorLoopPhis(loc)) {
return false;
}
// Create the generator resume index dispatch for the new loop header. After
// this, |current| is the normal non-resume loop body.
if (!entries.empty()) {
current->setHasGeneratorResumeEntry();
MBasicBlock* body = nullptr;
if (!startResumePath(current, loc, &body)) {
return false;
}
if (!buildResumeIndexDispatch(loc, entries)) {
return false;
}
current = body;
}
// Add the loop's interrupt check on the non-resume path. Interrupt checks can
// run arbitrary JS/Debugger code and handling this on the resume path is
// complicated because we're in the middle of restoring the frame.
MInterruptCheck* check = MInterruptCheck::New(alloc());
current->add(check);
#ifdef JS_CACHEIR_SPEW
if (snapshot().needsFinalWarmUpCount()) {
MIncrementWarmUpCounter* ins =
MIncrementWarmUpCounter::New(alloc(), script_);
current->add(ins);
}
#endif
return true;
}
bool WarpBuilder::buildTestOp(BytecodeLocation loc) {
MDefinition* originalValue = current->peek(-1);
if (auto* cacheIRSnapshot = getOpSnapshot<WarpCacheIR>(loc)) {
// If we have CacheIR, we can use it to refine the input. Note that
// the transpiler doesn't generate any control instructions. Instead,
// we fall through and generate them below.
MDefinition* value = current->pop();
if (!TranspileCacheIRToMIR(this, loc, cacheIRSnapshot, {value})) {
return false;
}
}
if (loc.isBackedge()) {
return buildTestBackedge(loc);
}
JSOp op = loc.getOp();
BytecodeLocation target1 = loc.next();
BytecodeLocation target2 = loc.getJumpTarget();
if (TestTrueTargetIsJoinPoint(op)) {
std::swap(target1, target2);
}
MDefinition* value = current->pop();
// JSOp::And and JSOp::Or leave the top stack value unchanged. The
// top stack value may have been converted to bool by a transpiled
// ToBool IC, so we push the original value.
bool mustKeepCondition = (op == JSOp::And || op == JSOp::Or);
if (mustKeepCondition) {
current->push(originalValue);
}
// If this op always branches to the same location we treat this as a
// JSOp::Goto.
if (target1 == target2) {
value->setImplicitlyUsedUnchecked();
return buildForwardGoto(target1);
}
MTest* test = MTest::New(alloc(), value, /* ifTrue = */ nullptr,
/* ifFalse = */ nullptr);
current->end(test);
// JSOp::Case must pop a second value on the true-branch (the input to the
// switch-statement).
uint32_t numToPop = (loc.getOp() == JSOp::Case) ? 1 : 0;
if (!addPendingEdge(target1, current, MTest::TrueBranchIndex, numToPop)) {
return false;
}
if (!addPendingEdge(target2, current, MTest::FalseBranchIndex)) {
return false;
}
if (const auto* typesSnapshot = getOpSnapshot<WarpPolymorphicTypes>(loc)) {
test->setObservedTypes(typesSnapshot->list());
}
setTerminatedBlock();
return true;
}
bool WarpBuilder::buildTestBackedge(BytecodeLocation loc) {
MOZ_ASSERT(loc.is(JSOp::JumpIfTrue));
MOZ_ASSERT(loopDepth() > 0);
MDefinition* value = current->pop();
BytecodeLocation loopHead = loc.getJumpTarget();
MOZ_ASSERT(loopHead.is(JSOp::LoopHead));
BytecodeLocation successor = loc.next();
// We can finish the loop now. Use the loophead pc instead of the current pc
// because the stack depth at the start of that op matches the current stack
// depth (after popping our operand).
MBasicBlock* pred = current;
if (!startNewBlock(current, loopHead)) {
return false;
}
MTest* test = MTest::New(alloc(), value, /* ifTrue = */ current,
/* ifFalse = */ nullptr);
pred->end(test);
if (const auto* typesSnapshot = getOpSnapshot<WarpPolymorphicTypes>(loc)) {
test->setObservedTypes(typesSnapshot->list());
}
if (!addPendingEdge(successor, pred, MTest::FalseBranchIndex)) {
return false;
}
return buildBackedge();
}
bool WarpBuilder::build_JumpIfFalse(BytecodeLocation loc) {
return buildTestOp(loc);
}
bool WarpBuilder::build_JumpIfTrue(BytecodeLocation loc) {
return buildTestOp(loc);
}
bool WarpBuilder::build_And(BytecodeLocation loc) { return buildTestOp(loc); }
bool WarpBuilder::build_Or(BytecodeLocation loc) { return buildTestOp(loc); }
bool WarpBuilder::build_Case(BytecodeLocation loc) { return buildTestOp(loc); }
bool WarpBuilder::build_Default(BytecodeLocation loc) {
current->pop();
return buildForwardGoto(loc.getJumpTarget());
}
bool WarpBuilder::build_Coalesce(BytecodeLocation loc) {
BytecodeLocation target1 = loc.next();
BytecodeLocation target2 = loc.getJumpTarget();
MOZ_ASSERT(target2 > target1);
MDefinition* value = current->peek(-1);
MInstruction* isNullOrUndefined = MIsNullOrUndefined::New(alloc(), value);
current->add(isNullOrUndefined);
current->end(MTest::New(alloc(), isNullOrUndefined, /* ifTrue = */ nullptr,
/* ifFalse = */ nullptr));
if (!addPendingEdge(target1, current, MTest::TrueBranchIndex)) {
return false;
}
if (!addPendingEdge(target2, current, MTest::FalseBranchIndex)) {
return false;
}
setTerminatedBlock();
return true;
}
bool WarpBuilder::buildBackedge() {
decLoopDepth();
MBasicBlock* header = loopStack_.popCopy().header();
current->end(MGoto::New(alloc(), header));
if (!header->setBackedge(current)) {
return false;
}
setTerminatedBlock();
return true;
}
bool WarpBuilder::buildForwardGoto(BytecodeLocation target) {
current->end(MGoto::New(alloc(), nullptr));
if (!addPendingEdge(target, current, MGoto::TargetIndex)) {
return false;
}
setTerminatedBlock();
return true;
}
bool WarpBuilder::build_Goto(BytecodeLocation loc) {
if (loc.isBackedge()) {
return buildBackedge();
}
return buildForwardGoto(loc.getJumpTarget());
}
bool WarpBuilder::build_IsNullOrUndefined(BytecodeLocation loc) {
MDefinition* value = current->peek(-1);
auto* isNullOrUndef = MIsNullOrUndefined::New(alloc(), value);
current->add(isNullOrUndef);
current->push(isNullOrUndef);
return true;
}
bool WarpBuilder::build_DebugCheckSelfHosted(BytecodeLocation loc) {
#ifdef DEBUG
MDefinition* val = current->pop();
MDebugCheckSelfHosted* check = MDebugCheckSelfHosted::New(alloc(), val);
current->add(check);
current->push(check);
if (!resumeAfter(check, loc)) {
return false;
}
#endif
return true;
}
bool WarpBuilder::build_DynamicImport(BytecodeLocation loc) {
auto phase = ImportPhase(GET_UINT8(loc.toRawBytecode()));
MDefinition* options = current->pop();
MDefinition* specifier = current->pop();
MDynamicImport* ins = MDynamicImport::New(alloc(), specifier, options, phase);
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
bool WarpBuilder::build_Not(BytecodeLocation loc) {
if (auto* cacheIRSnapshot = getOpSnapshot<WarpCacheIR>(loc)) {
// If we have CacheIR, we can use it to refine the input before
// emitting the MNot.
MDefinition* value = current->pop();
if (!TranspileCacheIRToMIR(this, loc, cacheIRSnapshot, {value})) {
return false;
}
}
MDefinition* value = current->pop();
MNot* ins = MNot::New(alloc(), value);
current->add(ins);
current->push(ins);
if (const auto* typesSnapshot = getOpSnapshot<WarpPolymorphicTypes>(loc)) {
ins->setObservedTypes(typesSnapshot->list());
}
return true;
}
bool WarpBuilder::build_ToString(BytecodeLocation loc) {
MDefinition* value = current->pop();
if (value->type() == MIRType::String) {
value->setImplicitlyUsedUnchecked();
current->push(value);
return true;
}
MToString* ins =
MToString::New(alloc(), value, MToString::SideEffectHandling::Supported);
current->add(ins);
current->push(ins);
if (ins->isEffectful()) {
return resumeAfter(ins, loc);
}
return true;
}
bool WarpBuilder::usesEnvironmentChain() const {
return script_->jitScript()->usesEnvironmentChain();
}
bool WarpBuilder::build_GlobalOrEvalDeclInstantiation(BytecodeLocation loc) {
MOZ_ASSERT(!script_->isForEval(), "Eval scripts not supported");
auto* redeclCheck = MGlobalDeclInstantiation::New(alloc());
current->add(redeclCheck);
return resumeAfter(redeclCheck, loc);
}
bool WarpBuilder::build_MutateProto(BytecodeLocation loc) {
MDefinition* value = current->pop();
MDefinition* obj = current->peek(-1);
MMutateProto* mutate = MMutateProto::New(alloc(), obj, value);
current->add(mutate);
return resumeAfter(mutate, loc);
}
MDefinition* WarpBuilder::getCallee() {
if (inlineCallInfo()) {
return inlineCallInfo()->callee();
}
MInstruction* callee = MCallee::New(alloc());
current->add(callee);
return callee;
}
bool WarpBuilder::build_Callee(BytecodeLocation) {
MDefinition* callee = getCallee();
current->push(callee);
return true;
}
bool WarpBuilder::build_ToAsyncIter(BytecodeLocation loc) {
MDefinition* nextMethod = current->pop();
MDefinition* iterator = current->pop();
MToAsyncIter* ins = MToAsyncIter::New(alloc(), iterator, nextMethod);
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
bool WarpBuilder::build_ToPropertyKey(BytecodeLocation loc) {
MDefinition* value = current->pop();
return buildIC(loc, CacheKind::ToPropertyKey, {value});
}
bool WarpBuilder::build_Typeof(BytecodeLocation loc) {
MDefinition* input = current->pop();
if (const auto* typesSnapshot = getOpSnapshot<WarpPolymorphicTypes>(loc)) {
auto* typeOf = MTypeOf::New(alloc(), input);
typeOf->setObservedTypes(typesSnapshot->list());
current->add(typeOf);
auto* ins = MTypeOfName::New(alloc(), typeOf);
current->add(ins);
current->push(ins);
return true;
}
return buildIC(loc, CacheKind::TypeOf, {input});
}
bool WarpBuilder::build_TypeofExpr(BytecodeLocation loc) {
return build_Typeof(loc);
}
bool WarpBuilder::build_TypeofEq(BytecodeLocation loc) {
auto operand = loc.getTypeofEqOperand();
JSType type = operand.type();
JSOp compareOp = operand.compareOp();
MDefinition* input = current->pop();
if (const auto* typesSnapshot = getOpSnapshot<WarpPolymorphicTypes>(loc)) {
auto* typeOf = MTypeOf::New(alloc(), input);
typeOf->setObservedTypes(typesSnapshot->list());
current->add(typeOf);
auto* typeInt = MConstant::NewInt32(alloc(), type);
current->add(typeInt);
auto* ins = MCompare::New(alloc(), typeOf, typeInt, compareOp,
MCompare::Compare_Int32);
current->add(ins);
current->push(ins);
return true;
}
return buildIC(loc, CacheKind::TypeOfEq, {input});
}
bool WarpBuilder::build_Arguments(BytecodeLocation loc) {
auto* snapshot = getOpSnapshot<WarpArguments>(loc);
MOZ_ASSERT(info().needsArgsObj());
MOZ_ASSERT(snapshot);
MOZ_ASSERT(usesEnvironmentChain());
ArgumentsObject* templateObj = snapshot->templateObj();
MDefinition* env = current->environmentChain();
MInstruction* argsObj;
if (inlineCallInfo()) {
argsObj = MCreateInlinedArgumentsObject::New(
alloc(), env, getCallee(), inlineCallInfo()->argv(), templateObj);
if (!argsObj) {
return false;
}
} else {
argsObj = MCreateArgumentsObject::New(alloc(), env, templateObj);
}
current->add(argsObj);
current->setArgumentsObject(argsObj);
current->push(argsObj);
return true;
}
bool WarpBuilder::build_ObjWithProto(BytecodeLocation loc) {
MDefinition* proto = current->pop();
MInstruction* ins = MObjectWithProto::New(alloc(), proto);
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
MDefinition* WarpBuilder::walkEnvironmentChain(uint32_t numHops) {
MDefinition* env = current->environmentChain();
for (uint32_t i = 0; i < numHops; i++) {
if (!alloc().ensureBallast()) {
return nullptr;
}
MInstruction* ins = MEnclosingEnvironment::New(alloc(), env);
current->add(ins);
env = ins;
}
return env;
}
bool WarpBuilder::build_GetAliasedVar(BytecodeLocation loc) {
EnvironmentCoordinate ec = loc.getEnvironmentCoordinate();
MDefinition* obj = walkEnvironmentChain(ec.hops());
if (!obj) {
return false;
}
MInstruction* load;
if (EnvironmentObject::nonExtensibleIsFixedSlot(ec)) {
load = MLoadFixedSlot::New(alloc(), obj, ec.slot());
} else {
MInstruction* slots = MSlots::New(alloc(), obj);
current->add(slots);
uint32_t slot = EnvironmentObject::nonExtensibleDynamicSlotIndex(ec);
load = MLoadDynamicSlot::New(alloc(), slots, slot);
}
current->add(load);
current->push(load);
return true;
}
bool WarpBuilder::build_SetAliasedVar(BytecodeLocation loc) {
EnvironmentCoordinate ec = loc.getEnvironmentCoordinate();
MDefinition* val = current->peek(-1);
MDefinition* obj = walkEnvironmentChain(ec.hops());
if (!obj) {
return false;
}
MInstruction* store;
if (EnvironmentObject::nonExtensibleIsFixedSlot(ec)) {
store = MStoreFixedSlot::NewBarriered(alloc(), obj, ec.slot(), val);
} else {
MInstruction* slots = MSlots::New(alloc(), obj);
current->add(slots);
uint32_t slot = EnvironmentObject::nonExtensibleDynamicSlotIndex(ec);
store = MStoreDynamicSlot::NewBarriered(alloc(), slots, slot, val);
}
current->add(store);
return resumeAfter(store, loc);
}
bool WarpBuilder::build_InitAliasedLexical(BytecodeLocation loc) {
return build_SetAliasedVar(loc);
}
bool WarpBuilder::build_EnvCallee(BytecodeLocation loc) {
uint32_t numHops = loc.getEnvCalleeNumHops();
MDefinition* env = walkEnvironmentChain(numHops);
if (!env) {
return false;
}
auto* callee = MLoadFixedSlot::New(alloc(), env, CallObject::calleeSlot());
current->add(callee);
current->push(callee);
return true;
}
bool WarpBuilder::build_Iter(BytecodeLocation loc) {
MDefinition* obj = current->pop();
return buildIC(loc, CacheKind::GetIterator, {obj});
}
bool WarpBuilder::build_MoreIter(BytecodeLocation loc) {
MDefinition* iter = current->peek(-1);
MInstruction* ins = MIteratorMore::New(alloc(), iter);
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
bool WarpBuilder::build_EndIter(BytecodeLocation loc) {
current->pop(); // Iterator value is not used.
MDefinition* iter = current->pop();
MInstruction* ins = MIteratorEnd::New(alloc(), iter);
current->add(ins);
return resumeAfter(ins, loc);
}
bool WarpBuilder::build_CloseIter(BytecodeLocation loc) {
MDefinition* iter = current->pop();
iter = unboxObjectInfallible(iter, IsMovable::No);
return buildIC(loc, CacheKind::CloseIter, {iter});
}
bool WarpBuilder::build_IsNoIter(BytecodeLocation) {
MDefinition* def = current->peek(-1);
MOZ_ASSERT(def->isIteratorMore());
MInstruction* ins = MIsNoIter::New(alloc(), def);
current->add(ins);
current->push(ins);
return true;
}
bool WarpBuilder::build_OptimizeGetIterator(BytecodeLocation loc) {
MDefinition* value = current->pop();
return buildIC(loc, CacheKind::OptimizeGetIterator, {value});
}
bool WarpBuilder::transpileCall(BytecodeLocation loc,
const WarpCacheIR* cacheIRSnapshot,
CallInfo* callInfo) {
// Synthesize the constant number of arguments for this call op.
auto* argc = MConstant::NewInt32(alloc(), callInfo->argc());
current->add(argc);
return TranspileCacheIRToMIR(this, loc, cacheIRSnapshot, {argc}, callInfo);
}
void WarpBuilder::buildCreateThis(CallInfo& callInfo) {
MOZ_ASSERT(callInfo.constructing());
// Inline the this-object allocation on the caller-side.
MDefinition* callee = callInfo.callee();
MDefinition* newTarget = callInfo.getNewTarget();
auto* createThis = MCreateThis::New(alloc(), callee, newTarget);
current->add(createThis);
callInfo.thisArg()->setImplicitlyUsedUnchecked();
callInfo.setThis(createThis);
}
bool WarpBuilder::buildCallOp(BytecodeLocation loc) {
uint32_t argc = loc.getCallArgc();
JSOp op = loc.getOp();
bool constructing = IsConstructOp(op);
bool ignoresReturnValue = (op == JSOp::CallIgnoresRv || loc.resultIsPopped());
CallInfo callInfo(alloc(), constructing, ignoresReturnValue);
if (!callInfo.init(current, argc)) {
return false;
}
if (const auto* inliningSnapshot = getOpSnapshot<WarpInlinedCall>(loc)) {
// Transpile the CacheIR to generate the correct guards before
// inlining. In this case, the call op (CallInlinedFunction,
// CallInlinedBoundFunction, or for monomorphic inlining the un-replaced
// CallScriptedFunction/CallBoundScriptedFunction) updates the CallInfo,
// but does not generate a call.
callInfo.markAsInlined();
if (!transpileCall(loc, inliningSnapshot->cacheIRSnapshot(), &callInfo)) {
return false;
}
// Generate the body of the inlined function.
return buildInlinedCall(loc, inliningSnapshot, callInfo);
}
if (auto* cacheIRSnapshot = getOpSnapshot<WarpCacheIR>(loc)) {
return transpileCall(loc, cacheIRSnapshot, &callInfo);
}
if (getOpSnapshot<WarpBailout>(loc)) {
callInfo.setImplicitlyUsedUnchecked();
return buildBailoutForColdIC(loc, CacheKind::Call);
}
bool needsThisCheck = false;
if (callInfo.constructing()) {
buildCreateThis(callInfo);
needsThisCheck = true;
}
MCall* call = makeCall(callInfo, needsThisCheck);
if (!call) {
return false;
}
current->add(call);
current->push(call);
return resumeAfter(call, loc);
}
bool WarpBuilder::build_Call(BytecodeLocation loc) { return buildCallOp(loc); }
bool WarpBuilder::build_CallContent(BytecodeLocation loc) {
return buildCallOp(loc);
}
bool WarpBuilder::build_CallIgnoresRv(BytecodeLocation loc) {
return buildCallOp(loc);
}
bool WarpBuilder::build_CallIter(BytecodeLocation loc) {
return buildCallOp(loc);
}
bool WarpBuilder::build_CallContentIter(BytecodeLocation loc) {
return buildCallOp(loc);
}
bool WarpBuilder::build_New(BytecodeLocation loc) { return buildCallOp(loc); }
bool WarpBuilder::build_NewContent(BytecodeLocation loc) {
return buildCallOp(loc);
}
bool WarpBuilder::build_SuperCall(BytecodeLocation loc) {
return buildCallOp(loc);
}
bool WarpBuilder::build_FunctionThis(BytecodeLocation loc) {
MOZ_ASSERT(info().hasFunMaybeLazy());
if (script_->strict()) {
// No need to wrap primitive |this| in strict mode.
current->pushSlot(info().thisSlot());
return true;
}
MOZ_ASSERT(!script_->hasNonSyntacticScope(),
"WarpOracle should have aborted compilation");
MDefinition* def = current->getSlot(info().thisSlot());
JSObject* globalThis = snapshot().globalLexicalEnvThis();
auto* thisObj = MBoxNonStrictThis::New(alloc(), def, globalThis);
current->add(thisObj);
current->push(thisObj);
return true;
}
bool WarpBuilder::build_GlobalThis(BytecodeLocation loc) {
MOZ_ASSERT(!script_->hasNonSyntacticScope());
JSObject* obj = snapshot().globalLexicalEnvThis();
pushConstant(ObjectValue(*obj));
return true;
}
MConstant* WarpBuilder::globalLexicalEnvConstant() {
JSObject* globalLexical = snapshot().globalLexicalEnv();
return constant(ObjectValue(*globalLexical));
}
bool WarpBuilder::build_GetName(BytecodeLocation loc) {
MOZ_ASSERT(usesEnvironmentChain());
MDefinition* env = current->environmentChain();
env = unboxObjectInfallible(env, IsMovable::Yes);
return buildIC(loc, CacheKind::GetName, {env});
}
bool WarpBuilder::build_GetGName(BytecodeLocation loc) {
MOZ_ASSERT(!script_->hasNonSyntacticScope());
MDefinition* env = globalLexicalEnvConstant();
return buildIC(loc, CacheKind::GetName, {env});
}
bool WarpBuilder::build_BindName(BytecodeLocation loc) {
MOZ_ASSERT(usesEnvironmentChain());
MDefinition* env = current->environmentChain();
env = unboxObjectInfallible(env, IsMovable::Yes);
return buildIC(loc, CacheKind::BindName, {env});
}
bool WarpBuilder::build_BindUnqualifiedName(BytecodeLocation loc) {
MOZ_ASSERT(usesEnvironmentChain());
MDefinition* env = current->environmentChain();
env = unboxObjectInfallible(env, IsMovable::Yes);
return buildIC(loc, CacheKind::BindName, {env});
}
bool WarpBuilder::build_BindUnqualifiedGName(BytecodeLocation loc) {
MOZ_ASSERT(!script_->hasNonSyntacticScope());
if (const auto* snapshot = getOpSnapshot<WarpBindUnqualifiedGName>(loc)) {
JSObject* globalEnv = snapshot->globalEnv();
pushConstant(ObjectValue(*globalEnv));
return true;
}
MDefinition* env = globalLexicalEnvConstant();
return buildIC(loc, CacheKind::BindName, {env});
}
bool WarpBuilder::build_GetProp(BytecodeLocation loc) {
MDefinition* val = current->pop();
return buildIC(loc, CacheKind::GetProp, {val});
}
bool WarpBuilder::build_GetBoundName(BytecodeLocation loc) {
MDefinition* val = current->pop();
return buildIC(loc, CacheKind::GetProp, {val});
}
bool WarpBuilder::build_GetElem(BytecodeLocation loc) {
MDefinition* id = current->pop();
MDefinition* val = current->pop();
return buildIC(loc, CacheKind::GetElem, {val, id});
}
bool WarpBuilder::build_SetProp(BytecodeLocation loc) {
MDefinition* val = current->pop();
MDefinition* obj = current->pop();
current->push(val);
return buildIC(loc, CacheKind::SetProp, {obj, val});
}
bool WarpBuilder::build_StrictSetProp(BytecodeLocation loc) {
return build_SetProp(loc);
}
bool WarpBuilder::build_SetName(BytecodeLocation loc) {
return build_SetProp(loc);
}
bool WarpBuilder::build_StrictSetName(BytecodeLocation loc) {
return build_SetProp(loc);
}
bool WarpBuilder::build_SetGName(BytecodeLocation loc) {
return build_SetProp(loc);
}
bool WarpBuilder::build_StrictSetGName(BytecodeLocation loc) {
return build_SetProp(loc);
}
bool WarpBuilder::build_InitGLexical(BytecodeLocation loc) {
MOZ_ASSERT(!script_->hasNonSyntacticScope());
MDefinition* globalLexical = globalLexicalEnvConstant();
MDefinition* val = current->peek(-1);
return buildIC(loc, CacheKind::SetProp, {globalLexical, val});
}
bool WarpBuilder::build_SetElem(BytecodeLocation loc) {
MDefinition* val = current->pop();
MDefinition* id = current->pop();
MDefinition* obj = current->pop();
current->push(val);
return buildIC(loc, CacheKind::SetElem, {obj, id, val});
}
bool WarpBuilder::build_StrictSetElem(BytecodeLocation loc) {
return build_SetElem(loc);
}
bool WarpBuilder::build_DelProp(BytecodeLocation loc) {
PropertyName* name = loc.getPropertyName(script_);
MDefinition* obj = current->pop();
bool strict = loc.getOp() == JSOp::StrictDelProp;
MInstruction* ins = MDeleteProperty::New(alloc(), obj, name, strict);
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
bool WarpBuilder::build_StrictDelProp(BytecodeLocation loc) {
return build_DelProp(loc);
}
bool WarpBuilder::build_DelElem(BytecodeLocation loc) {
MDefinition* id = current->pop();
MDefinition* obj = current->pop();
bool strict = loc.getOp() == JSOp::StrictDelElem;
MInstruction* ins = MDeleteElement::New(alloc(), obj, id, strict);
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
bool WarpBuilder::build_StrictDelElem(BytecodeLocation loc) {
return build_DelElem(loc);
}
bool WarpBuilder::build_SetFunName(BytecodeLocation loc) {
FunctionPrefixKind prefixKind = loc.getFunctionPrefixKind();
MDefinition* name = current->pop();
MDefinition* fun = current->pop();
MSetFunName* ins = MSetFunName::New(alloc(), fun, name, uint8_t(prefixKind));
current->add(ins);
current->push(fun);
return resumeAfter(ins, loc);
}
bool WarpBuilder::build_PushLexicalEnv(BytecodeLocation loc) {
MOZ_ASSERT(usesEnvironmentChain());
const auto* snapshot = getOpSnapshot<WarpLexicalEnvironment>(loc);
MOZ_ASSERT(snapshot);
MDefinition* env = current->environmentChain();
MConstant* templateCst = constant(ObjectValue(*snapshot->templateObj()));
auto* ins = MNewLexicalEnvironmentObject::New(alloc(), templateCst);
current->add(ins);
// Initialize the object's reserved slots. No post barrier is needed here,
// for the same reason as in buildNamedLambdaEnv.
auto* store = MStoreFixedSlot::NewNoPreBarrier(
alloc(), ins, EnvironmentObject::enclosingEnvironmentSlot(), env);
store->setNeedsPostBarrier(false);
current->add(store);
current->setEnvironmentChain(ins);
return true;
}
bool WarpBuilder::build_PushClassBodyEnv(BytecodeLocation loc) {
MOZ_ASSERT(usesEnvironmentChain());
const auto* snapshot = getOpSnapshot<WarpClassBodyEnvironment>(loc);
MOZ_ASSERT(snapshot);
MDefinition* env = current->environmentChain();
MConstant* templateCst = constant(ObjectValue(*snapshot->templateObj()));
auto* ins = MNewClassBodyEnvironmentObject::New(alloc(), templateCst);
current->add(ins);
// Initialize the object's reserved slots. No post barrier is needed here,
// for the same reason as in buildNamedLambdaEnv.
auto* store = MStoreFixedSlot::NewNoPreBarrier(
alloc(), ins, EnvironmentObject::enclosingEnvironmentSlot(), env);
store->setNeedsPostBarrier(false);
current->add(store);
current->setEnvironmentChain(ins);
return true;
}
bool WarpBuilder::build_PopLexicalEnv(BytecodeLocation) {
MDefinition* enclosingEnv = walkEnvironmentChain(1);
if (!enclosingEnv) {
return false;
}
current->setEnvironmentChain(enclosingEnv);
return true;
}
bool WarpBuilder::build_FreshenLexicalEnv(BytecodeLocation loc) {
MOZ_ASSERT(usesEnvironmentChain());
const auto* snapshot = getOpSnapshot<WarpLexicalEnvironment>(loc);
MOZ_ASSERT(snapshot);
MDefinition* enclosingEnv = walkEnvironmentChain(1);
if (!enclosingEnv) {
return false;
}
MDefinition* env = current->environmentChain();
MConstant* templateCst = constant(ObjectValue(*snapshot->templateObj()));
auto* templateObj = snapshot->templateObj();
auto* scope = &templateObj->scope();
MOZ_ASSERT(scope->hasEnvironment());
auto* ins = MNewLexicalEnvironmentObject::New(alloc(), templateCst);
current->add(ins);
// Initialize the object's reserved slots. No post barrier is needed here,
// for the same reason as in buildNamedLambdaEnv.
auto* store = MStoreFixedSlot::NewNoPreBarrier(
alloc(), ins, EnvironmentObject::enclosingEnvironmentSlot(),
enclosingEnv);
store->setNeedsPostBarrier(false);
current->add(store);
// Copy environment slots.
MSlots* envSlots = nullptr;
MSlots* slots = nullptr;
for (BindingIter iter(scope); iter; iter++) {
auto loc = iter.location();
if (loc.kind() != BindingLocation::Kind::Environment) {
MOZ_ASSERT(loc.kind() == BindingLocation::Kind::Frame);
continue;
}
if (!alloc().ensureBallast()) {
return false;
}
uint32_t slot = loc.slot();
uint32_t numFixedSlots = templateObj->numFixedSlots();
if (slot >= numFixedSlots) {
if (!envSlots) {
envSlots = MSlots::New(alloc(), env);
current->add(envSlots);
}
if (!slots) {
slots = MSlots::New(alloc(), ins);
current->add(slots);
}
uint32_t dynamicSlot = slot - numFixedSlots;
auto* load = MLoadDynamicSlot::New(alloc(), envSlots, dynamicSlot);
current->add(load);
auto* store =
MStoreDynamicSlot::NewNoPreBarrier(alloc(), slots, dynamicSlot, load);
store->setNeedsPostBarrier(false);
current->add(store);
} else {
auto* load = MLoadFixedSlot::New(alloc(), env, slot);
current->add(load);
auto* store = MStoreFixedSlot::NewNoPreBarrier(alloc(), ins, slot, load);
store->setNeedsPostBarrier(false);
current->add(store);
}
}
current->setEnvironmentChain(ins);
return true;
}
bool WarpBuilder::build_RecreateLexicalEnv(BytecodeLocation loc) {
MOZ_ASSERT(usesEnvironmentChain());
const auto* snapshot = getOpSnapshot<WarpLexicalEnvironment>(loc);
MOZ_ASSERT(snapshot);
MDefinition* enclosingEnv = walkEnvironmentChain(1);
if (!enclosingEnv) {
return false;
}
MConstant* templateCst = constant(ObjectValue(*snapshot->templateObj()));
auto* ins = MNewLexicalEnvironmentObject::New(alloc(), templateCst);
current->add(ins);
// Initialize the object's reserved slots. No post barrier is needed here,
// for the same reason as in buildNamedLambdaEnv.
auto* store = MStoreFixedSlot::NewNoPreBarrier(
alloc(), ins, EnvironmentObject::enclosingEnvironmentSlot(),
enclosingEnv);
store->setNeedsPostBarrier(false);
current->add(store);
current->setEnvironmentChain(ins);
return true;
}
bool WarpBuilder::build_PushVarEnv(BytecodeLocation loc) {
MOZ_ASSERT(usesEnvironmentChain());
const auto* snapshot = getOpSnapshot<WarpVarEnvironment>(loc);
MOZ_ASSERT(snapshot);
MDefinition* env = current->environmentChain();
MConstant* templateCst = constant(ObjectValue(*snapshot->templateObj()));
auto* ins = MNewVarEnvironmentObject::New(alloc(), templateCst);
current->add(ins);
// Initialize the object's reserved slots. No post barrier is needed here,
// for the same reason as in buildNamedLambdaEnv.
auto* store = MStoreFixedSlot::NewNoPreBarrier(
alloc(), ins, EnvironmentObject::enclosingEnvironmentSlot(), env);
store->setNeedsPostBarrier(false);
current->add(store);
current->setEnvironmentChain(ins);
return true;
}
bool WarpBuilder::build_ImplicitThis(BytecodeLocation loc) {
MDefinition* env = current->pop();
auto* ins = MImplicitThis::New(alloc(), env);
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
bool WarpBuilder::build_CheckClassHeritage(BytecodeLocation loc) {
MDefinition* def = current->pop();
auto* ins = MCheckClassHeritage::New(alloc(), def);
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
bool WarpBuilder::build_CheckThis(BytecodeLocation loc) {
MDefinition* def = current->pop();
auto* ins = MCheckThis::New(alloc(), def);
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
bool WarpBuilder::build_CheckThisReinit(BytecodeLocation loc) {
MDefinition* def = current->pop();
auto* ins = MCheckThisReinit::New(alloc(), def);
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
bool WarpBuilder::build_Generator(BytecodeLocation loc) {
MOZ_ASSERT(usesEnvironmentChain());
MDefinition* callee = getCallee();
MDefinition* environmentChain = current->environmentChain();
MDefinition* argsObj = info().needsArgsObj() ? current->argumentsObject()
: constant(Int32Value(0));
MGenerator* generator =
MGenerator::New(alloc(), callee, environmentChain, argsObj);
current->add(generator);
current->push(generator);
return resumeAfter(generator, loc);
}
bool WarpBuilder::build_AfterYield(BytecodeLocation loc) {
MOZ_ASSERT(resumeAnalysis_.isSome());
// The preceding block was terminated by the suspend. Get the stack depth
// saved by the suspend op.
MOZ_ASSERT(hasTerminatedBlock());
mozilla::Maybe<uint32_t> savedSlots = pendingSuspendSavedSlots_.take();
MOZ_ASSERT_IF(savedSlots, pendingSuspendResumeIndex_ ==
loc.getSuspendForAfterYield().getResumeIndex());
// If the AfterYield is in a loop we didn't compile because it was
// unreachable, there will be no dispatch entry.
// See BailoutKind::UncompiledGeneratorResume.
PendingEdgesMap::Ptr p = pendingEdges_.lookup(loc.toRawBytecode());
if (!p) {
MOZ_ASSERT(savedSlots.isNothing());
return true;
}
// There must be a single edge from the innermost resume index dispatch.
// AfterYield is a jump target op but is only targeted by the dispatch block.
PendingEdges edges(std::move(p->value()));
pendingEdges_.remove(p);
MOZ_RELEASE_ASSERT(edges.length() == 1);
// Create a new block with the dispatch block's pc. The values in its entry
// resume point are never used, because a bailout on the resume path redoes
// the resume in Baseline, but its stack depth must match its pc.
const PendingEdge& edge = edges[0];
BytecodeLocation predLoc(script_, edge.block()->pc());
if (!startNewBlock(edge.block(), predLoc, edge.numToPop())) {
return false;
}
edge.block()->lastIns()->initSuccessor(edge.successor(), current);
if (savedSlots.isNothing()) {
// We didn't compile the matching suspend because it was unreachable or in a
// catch-block. Bail out and resume in Baseline instead.
current->add(MBail::New(alloc(), BailoutKind::UncompiledGeneratorResume));
current->end(MUnreachable::New(alloc()));
setTerminatedBlock();
return true;
}
// Drop the loop header expression stack inherited from the resume path (the
// |undefined| placeholders pushed in buildLoopResumeMerge) before rebuilding
// the real one.
while (current->stackDepth() > info().firstStackSlot()) {
current->pop();
}
MDefinition* genObj = resumeGeneratorObject();
uint32_t nfixed = info().nlocals();
MOZ_ASSERT(*savedSlots >= nfixed);
uint32_t nexpr = *savedSlots - nfixed;
if (*savedSlots > 0) {
// Load the generator's stack storage elements.
MDefinition* arrayObj = loadGeneratorStackStorage(genObj);
auto* elements = MElements::New(alloc(), arrayObj);
current->add(elements);
// Restore the expression stack slots.
for (uint32_t i = 0; i < nexpr; i++) {
if (!alloc().ensureBallast()) {
return false;
}
auto* load =
MLoadElement::New(alloc(), elements, constant(Int32Value(nfixed + i)),
/* needsHoleCheck = */ false);
current->add(load);
current->push(load);
}
// Clear the stack storage array by setting its initialized length to 0.
// Nothing between here and the MClearResumingGeneratorFlag below may bail
// out: a bailout there would redo the resume in Baseline, which restores
// the frame from the stack storage we just drained.
// See BaselineStackBuilder::isGeneratorResumePrologueBailout.
current->add(MSetInitializedLength::New(alloc(), elements, 0,
/* needsPreBarrier = */ true));
}
// Push the [rval, resumeKind] values on top of the stack.
current->push(resumeFrameArg(ResumeFrameArgs::ResumeValueSlot));
auto* resumeKindVal = resumeFrameArg(ResumeFrameArgs::ResumeKindSlot);
auto* resumeKind = MUnbox::New(alloc(), resumeKindVal, MIRType::Int32,
MUnbox::Mode::Infallible);
current->add(resumeKind);
current->push(resumeKind);
// Done! End the resume path by clearing the IsResuming frame descriptor bit.
// From now on, loop backedges will take the normal path and the
// ResumeFrameArgs are dead.
auto* clear = MClearResumingGeneratorFlag::New(alloc());
current->add(clear);
return resumeAfter(clear, loc);
}
bool WarpBuilder::build_FinalYieldRval(BytecodeLocation loc) {
MDefinition* gen = current->pop();
auto setSlotNull = [this, gen](size_t slot) {
auto* ins = MStoreFixedSlot::NewBarriered(alloc(), gen, slot,
constant(NullValue()));
current->add(ins);
};
// Close the generator
setSlotNull(AbstractGeneratorObject::calleeOrModuleSlot());
setSlotNull(AbstractGeneratorObject::envChainSlot());
setSlotNull(AbstractGeneratorObject::argsObjectSlot());
setSlotNull(AbstractGeneratorObject::stackStorageSlot());
setSlotNull(AbstractGeneratorObject::resumeIndexSlot());
// Return
return build_RetRval(loc);
}
bool WarpBuilder::build_Resume(BytecodeLocation loc) {
MDefinition* resumeKind = current->pop();
MDefinition* value = current->pop();
MDefinition* gen = current->pop();
// resumeKind is always a constant Int32 emitted by JSOp::ResumeKind right
// before the resume.
MOZ_RELEASE_ASSERT(resumeKind->isConstant());
int32_t resumeKindInt32 = resumeKind->toConstant()->toInt32();
resumeKind->setImplicitlyUsedUnchecked();
auto* resume = MGeneratorResume::New(alloc(), gen, value, resumeKindInt32);
current->add(resume);
current->push(resume);
return resumeAfter(resume, loc);
}
bool WarpBuilder::build_AsyncResolve(BytecodeLocation loc) {
MDefinition* generator = current->pop();
MDefinition* value = current->pop();
auto* resolve = MAsyncResolve::New(alloc(), generator, value);
current->add(resolve);
current->push(resolve);
return resumeAfter(resolve, loc);
}
bool WarpBuilder::build_ResumeKind(BytecodeLocation loc) {
GeneratorResumeKind resumeKind = loc.resumeKind();
current->push(constant(Int32Value(static_cast<int32_t>(resumeKind))));
return true;
}
bool WarpBuilder::build_CanSkipAwait(BytecodeLocation loc) {
// LCanSkipAwait's frame descriptor check is not correct for inlined
// functions.
MOZ_ASSERT(!inlineCallInfo());
MDefinition* val = current->pop();
MCanSkipAwait* canSkip = MCanSkipAwait::New(alloc(), val);
current->add(canSkip);
current->push(val);
current->push(canSkip);
return resumeAfter(canSkip, loc);
}
bool WarpBuilder::build_MaybeExtractAwaitValue(BytecodeLocation loc) {
MDefinition* canSkip = current->pop();
MDefinition* value = current->pop();
MMaybeExtractAwaitValue* extracted =
MMaybeExtractAwaitValue::New(alloc(), value, canSkip);
current->add(extracted);
current->push(extracted);
current->push(canSkip);
return resumeAfter(extracted, loc);
}
bool WarpBuilder::build_InitialYield(BytecodeLocation loc) {
MDefinition* gen = current->pop();
return buildSuspend(loc, gen, gen);
}
bool WarpBuilder::build_Await(BytecodeLocation loc) {
MDefinition* gen = current->pop();
MDefinition* promiseOrGenerator = current->pop();
return buildSuspend(loc, gen, promiseOrGenerator);
}
bool WarpBuilder::build_Yield(BytecodeLocation loc) { return build_Await(loc); }
bool WarpBuilder::buildSuspend(BytecodeLocation loc, MDefinition* gen,
MDefinition* retVal) {
MOZ_ASSERT(resumeAnalysis_.isSome());
// If required, unbox the generator object explicitly and infallibly.
//
// This is done to avoid fuzz-bugs where ApplyTypeInformation does the
// unboxing, and generates fallible unboxes which can lead to torn object
// state due to `bailAfter`.
MDefinition* genObj = gen;
if (genObj->type() != MIRType::Object) {
auto* unbox =
MUnbox::New(alloc(), gen, MIRType::Object, MUnbox::Mode::Infallible);
current->add(unbox);
genObj = unbox;
}
int32_t slotsToCopy = current->stackDepth() - info().firstLocalSlot();
MOZ_ASSERT(slotsToCopy >= 0);
// Record the stack depth for the JSOp::AfterYield that follows.
MOZ_ASSERT(pendingSuspendSavedSlots_.isNothing());
pendingSuspendSavedSlots_.emplace(uint32_t(slotsToCopy));
#ifdef DEBUG
pendingSuspendResumeIndex_ = loc.getResumeIndex();
#endif
if (slotsToCopy > 0) {
MDefinition* arrayObj = loadGeneratorStackStorage(genObj);
auto* elements = MElements::New(alloc(), arrayObj);
current->add(elements);
for (int32_t i = 0; i < slotsToCopy; i++) {
if (!alloc().ensureBallast()) {
return false;
}
// Use peekUnchecked because we're also writing out the argument slots
int32_t peek = -slotsToCopy + i;
MDefinition* stackElem = current->peekUnchecked(peek);
auto* store = MStoreElement::NewNoPreBarrier(
alloc(), elements, constant(Int32Value(i)), stackElem,
/* needsHoleCheck = */ false);
current->add(store);
}
auto* setInitLength =
MSetInitializedLength::New(alloc(), elements, slotsToCopy,
/* needsPreBarrier = */ false);
current->add(setInitLength);
auto* setLength = MSetArrayLength::New(alloc(), elements, slotsToCopy);
current->add(setLength);
}
// Update Generator Object state
uint32_t resumeIndex = loc.getResumeIndex();
// This store is unbarriered, as it's only ever storing an integer, and as
// such doesn't partake of object tracing.
current->add(MStoreFixedSlot::NewNoPreBarrier(
alloc(), genObj, AbstractGeneratorObject::resumeIndexSlot(),
constant(Int32Value(resumeIndex))));
// This store is barriered because it stores an object value.
current->add(MStoreFixedSlot::NewBarriered(
alloc(), genObj, AbstractGeneratorObject::envChainSlot(),
current->environmentChain()));
// End the block with a Return. The AfterYield op will create a new block.
current->end(MReturn::New(alloc(), retVal));
if (!graph().addReturn(current)) {
return false;
}
setTerminatedBlock();
return true;
}
bool WarpBuilder::build_AsyncAwait(BytecodeLocation loc) {
MDefinition* gen = current->pop();
MDefinition* value = current->pop();
MAsyncAwait* asyncAwait = MAsyncAwait::New(alloc(), value, gen);
current->add(asyncAwait);
current->push(asyncAwait);
return resumeAfter(asyncAwait, loc);
}
bool WarpBuilder::build_CheckReturn(BytecodeLocation loc) {
MOZ_ASSERT(!script_->noScriptRval());
MDefinition* returnValue = current->getSlot(info().returnValueSlot());
MDefinition* thisValue = current->pop();
auto* ins = MCheckReturn::New(alloc(), returnValue, thisValue);
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
void WarpBuilder::buildCheckLexicalOp(BytecodeLocation loc) {
JSOp op = loc.getOp();
MOZ_ASSERT(op == JSOp::CheckLexical || op == JSOp::CheckAliasedLexical);
MDefinition* input = current->pop();
MInstruction* lexicalCheck = MLexicalCheck::New(alloc(), input);
current->add(lexicalCheck);
current->push(lexicalCheck);
if (snapshot().bailoutInfo().failedLexicalCheck()) {
// If we have previously had a failed lexical check in Ion, we want to avoid
// hoisting any lexical checks, which can cause spurious failures. In this
// case, we also have to be careful not to hoist any loads of this lexical
// past the check. For unaliased lexical variables, we can set the local
// slot to create a dependency (see below). For aliased lexicals, that
// doesn't work, so we disable LICM instead.
lexicalCheck->setNotMovable();
if (op == JSOp::CheckAliasedLexical) {
mirGen().disableLICM();
}
}
if (op == JSOp::CheckLexical) {
// Set the local slot so that a subsequent GetLocal without a CheckLexical
// (the frontend can elide lexical checks) doesn't let a definition with
// MIRType::MagicUninitializedLexical escape to arbitrary MIR instructions.
// Note that in this case the GetLocal would be unreachable because we throw
// an exception here, but we still generate MIR instructions for it.
uint32_t slot = info().localSlot(loc.local());
current->setSlot(slot, lexicalCheck);
}
}
bool WarpBuilder::build_CheckLexical(BytecodeLocation loc) {
buildCheckLexicalOp(loc);
return true;
}
bool WarpBuilder::build_CheckAliasedLexical(BytecodeLocation loc) {
buildCheckLexicalOp(loc);
return true;
}
bool WarpBuilder::build_InitHomeObject(BytecodeLocation loc) {
MDefinition* homeObject = current->pop();
MDefinition* function = current->pop();
auto* ins = MInitHomeObject::New(alloc(), function, homeObject);
current->add(ins);
current->push(ins);
return true;
}
bool WarpBuilder::build_SuperBase(BytecodeLocation) {
MDefinition* callee = current->pop();
auto* homeObject = MHomeObject::New(alloc(), callee);
current->add(homeObject);
auto* superBase = MHomeObjectSuperBase::New(alloc(), homeObject);
current->add(superBase);
current->push(superBase);
return true;
}
bool WarpBuilder::build_SuperFun(BytecodeLocation) {
MDefinition* callee = current->pop();
auto* ins = MSuperFunction::New(alloc(), callee);
current->add(ins);
current->push(ins);
return true;
}
bool WarpBuilder::build_BuiltinObject(BytecodeLocation loc) {
if (auto* snapshot = getOpSnapshot<WarpBuiltinObject>(loc)) {
JSObject* builtin = snapshot->builtin();
pushConstant(ObjectValue(*builtin));
return true;
}
auto kind = loc.getBuiltinObjectKind();
auto* ins = MBuiltinObject::New(alloc(), kind);
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
bool WarpBuilder::build_GetIntrinsic(BytecodeLocation loc) {
if (auto* snapshot = getOpSnapshot<WarpGetIntrinsic>(loc)) {
Value intrinsic = snapshot->intrinsic();
pushConstant(intrinsic);
return true;
}
PropertyName* name = loc.getPropertyName(script_);
MCallGetIntrinsicValue* ins = MCallGetIntrinsicValue::New(alloc(), name);
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
bool WarpBuilder::build_ImportMeta(BytecodeLocation loc) {
ModuleObject* moduleObj = scriptSnapshot()->moduleObject();
MOZ_ASSERT(moduleObj);
MModuleMetadata* ins = MModuleMetadata::New(alloc(), moduleObj);
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
bool WarpBuilder::build_CallSiteObj(BytecodeLocation loc) {
return build_Object(loc);
}
bool WarpBuilder::build_NewArray(BytecodeLocation loc) {
return buildIC(loc, CacheKind::NewArray, {});
}
bool WarpBuilder::build_NewObject(BytecodeLocation loc) {
return buildIC(loc, CacheKind::NewObject, {});
}
bool WarpBuilder::build_NewInit(BytecodeLocation loc) {
return build_NewObject(loc);
}
bool WarpBuilder::build_Object(BytecodeLocation loc) {
JSObject* obj = loc.getObject(script_);
MConstant* objConst = constant(ObjectValue(*obj));
current->push(objConst);
return true;
}
bool WarpBuilder::buildInitPropGetterSetterOp(BytecodeLocation loc) {
PropertyName* name = loc.getPropertyName(script_);
MDefinition* value = current->pop();
MDefinition* obj = current->peek(-1);
auto* ins = MInitPropGetterSetter::New(alloc(), obj, value, name);
current->add(ins);
return resumeAfter(ins, loc);
}
bool WarpBuilder::build_InitPropGetter(BytecodeLocation loc) {
return buildInitPropGetterSetterOp(loc);
}
bool WarpBuilder::build_InitPropSetter(BytecodeLocation loc) {
return buildInitPropGetterSetterOp(loc);
}
bool WarpBuilder::build_InitHiddenPropGetter(BytecodeLocation loc) {
return buildInitPropGetterSetterOp(loc);
}
bool WarpBuilder::build_InitHiddenPropSetter(BytecodeLocation loc) {
return buildInitPropGetterSetterOp(loc);
}
bool WarpBuilder::buildInitElemGetterSetterOp(BytecodeLocation loc) {
MDefinition* value = current->pop();
MDefinition* id = current->pop();
MDefinition* obj = current->peek(-1);
auto* ins = MInitElemGetterSetter::New(alloc(), obj, id, value);
current->add(ins);
return resumeAfter(ins, loc);
}
bool WarpBuilder::build_InitElemGetter(BytecodeLocation loc) {
return buildInitElemGetterSetterOp(loc);
}
bool WarpBuilder::build_InitElemSetter(BytecodeLocation loc) {
return buildInitElemGetterSetterOp(loc);
}
bool WarpBuilder::build_InitHiddenElemGetter(BytecodeLocation loc) {
return buildInitElemGetterSetterOp(loc);
}
bool WarpBuilder::build_InitHiddenElemSetter(BytecodeLocation loc) {
return buildInitElemGetterSetterOp(loc);
}
bool WarpBuilder::build_In(BytecodeLocation loc) {
MDefinition* obj = current->pop();
MDefinition* id = current->pop();
return buildIC(loc, CacheKind::In, {id, obj});
}
bool WarpBuilder::build_HasOwn(BytecodeLocation loc) {
MDefinition* obj = current->pop();
MDefinition* id = current->pop();
return buildIC(loc, CacheKind::HasOwn, {id, obj});
}
bool WarpBuilder::build_CheckPrivateField(BytecodeLocation loc) {
MDefinition* id = current->peek(-1);
MDefinition* obj = current->peek(-2);
return buildIC(loc, CacheKind::CheckPrivateField, {obj, id});
}
bool WarpBuilder::build_NewPrivateName(BytecodeLocation loc) {
JSAtom* name = loc.getAtom(script_);
auto* ins = MNewPrivateName::New(alloc(), &name->asOffThreadAtom());
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
bool WarpBuilder::build_Instanceof(BytecodeLocation loc) {
MDefinition* rhs = current->pop();
MDefinition* obj = current->pop();
return buildIC(loc, CacheKind::InstanceOf, {obj, rhs});
}
bool WarpBuilder::build_NewTarget(BytecodeLocation loc) {
MOZ_ASSERT(script_->isFunction());
MOZ_ASSERT(info().hasFunMaybeLazy());
MOZ_ASSERT(!scriptSnapshot()->isArrowFunction());
if (inlineCallInfo()) {
if (inlineCallInfo()->constructing()) {
current->push(inlineCallInfo()->getNewTarget());
} else {
pushConstant(UndefinedValue());
}
return true;
}
MNewTarget* ins = MNewTarget::New(alloc());
current->add(ins);
current->push(ins);
return true;
}
bool WarpBuilder::build_CheckIsObj(BytecodeLocation loc) {
CheckIsObjectKind kind = loc.getCheckIsObjectKind();
MDefinition* toCheck = current->peek(-1);
if (toCheck->type() == MIRType::Object) {
toCheck->setImplicitlyUsedUnchecked();
return true;
}
MDefinition* val = current->pop();
MCheckIsObj* ins = MCheckIsObj::New(alloc(), val, uint8_t(kind));
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
bool WarpBuilder::build_CheckObjCoercible(BytecodeLocation loc) {
MDefinition* val = current->pop();
MCheckObjCoercible* ins = MCheckObjCoercible::New(alloc(), val);
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
MInstruction* WarpBuilder::buildLoadSlot(MDefinition* obj,
uint32_t numFixedSlots,
uint32_t slot) {
if (slot < numFixedSlots) {
MLoadFixedSlot* load = MLoadFixedSlot::New(alloc(), obj, slot);
current->add(load);
return load;
}
MSlots* slots = MSlots::New(alloc(), obj);
current->add(slots);
MLoadDynamicSlot* load =
MLoadDynamicSlot::New(alloc(), slots, slot - numFixedSlots);
current->add(load);
return load;
}
bool WarpBuilder::build_GetImport(BytecodeLocation loc) {
auto* snapshot = getOpSnapshot<WarpGetImport>(loc);
ModuleEnvironmentObject* targetEnv = snapshot->targetEnv();
// Load the target environment slot.
MConstant* obj = constant(ObjectValue(*targetEnv));
auto* load = buildLoadSlot(obj, snapshot->numFixedSlots(), snapshot->slot());
if (snapshot->needsLexicalCheck()) {
// TODO: IonBuilder has code to mark non-movable. See buildCheckLexicalOp.
MInstruction* lexicalCheck = MLexicalCheck::New(alloc(), load);
current->add(lexicalCheck);
current->push(lexicalCheck);
} else {
current->push(load);
}
return true;
}
bool WarpBuilder::build_GetPropSuper(BytecodeLocation loc) {
MDefinition* obj = current->pop();
MDefinition* receiver = current->pop();
return buildIC(loc, CacheKind::GetPropSuper, {obj, receiver});
}
bool WarpBuilder::build_GetElemSuper(BytecodeLocation loc) {
MDefinition* obj = current->pop();
MDefinition* id = current->pop();
MDefinition* receiver = current->pop();
return buildIC(loc, CacheKind::GetElemSuper, {obj, id, receiver});
}
bool WarpBuilder::build_InitProp(BytecodeLocation loc) {
MDefinition* val = current->pop();
MDefinition* obj = current->peek(-1);
return buildIC(loc, CacheKind::SetProp, {obj, val});
}
bool WarpBuilder::build_InitLockedProp(BytecodeLocation loc) {
return build_InitProp(loc);
}
bool WarpBuilder::build_InitHiddenProp(BytecodeLocation loc) {
return build_InitProp(loc);
}
bool WarpBuilder::build_InitElem(BytecodeLocation loc) {
MDefinition* val = current->pop();
MDefinition* id = current->pop();
MDefinition* obj = current->peek(-1);
return buildIC(loc, CacheKind::SetElem, {obj, id, val});
}
bool WarpBuilder::build_InitLockedElem(BytecodeLocation loc) {
return build_InitElem(loc);
}
bool WarpBuilder::build_InitHiddenElem(BytecodeLocation loc) {
return build_InitElem(loc);
}
bool WarpBuilder::build_InitElemArray(BytecodeLocation loc) {
MDefinition* val = current->pop();
MDefinition* obj = current->peek(-1);
// Note: getInitElemArrayIndex asserts the index fits in int32_t.
uint32_t index = loc.getInitElemArrayIndex();
MConstant* indexConst = constant(Int32Value(index));
// Note: InitArrayElemOperation asserts the index does not exceed the array's
// dense element capacity.
auto* elements = MElements::New(alloc(), obj);
current->add(elements);
if (val->type() == MIRType::MagicHole) {
val->setImplicitlyUsedUnchecked();
auto* store = MStoreHoleValueElement::New(alloc(), elements, indexConst);
current->add(store);
} else {
auto* store =
MStoreElement::NewNoPreBarrier(alloc(), elements, indexConst, val,
/* needsHoleCheck = */ false);
current->add(store);
}
auto* setLength = MSetInitializedLength::New(alloc(), elements, index + 1,
/* needsPreBarrier = */ false);
current->add(setLength);
return resumeAfter(setLength, loc);
}
bool WarpBuilder::build_InitElemInc(BytecodeLocation loc) {
MDefinition* val = current->pop();
MDefinition* index = current->pop();
MDefinition* obj = current->peek(-1);
// Push index + 1.
MConstant* constOne = constant(Int32Value(1));
MAdd* nextIndex = MAdd::New(alloc(), index, constOne, TruncateKind::Truncate);
current->add(nextIndex);
current->push(nextIndex);
return buildIC(loc, CacheKind::SetElem, {obj, index, val});
}
bool WarpBuilder::build_Lambda(BytecodeLocation loc) {
MOZ_ASSERT(usesEnvironmentChain());
return buildIC(loc, CacheKind::Lambda, {});
}
bool WarpBuilder::build_FunWithProto(BytecodeLocation loc) {
MOZ_ASSERT(usesEnvironmentChain());
MDefinition* proto = current->pop();
MDefinition* env = current->environmentChain();
JSFunction* fun = loc.getFunction(script_);
MConstant* funConst = constant(ObjectValue(*fun));
auto* ins = MFunctionWithProto::New(alloc(), env, proto, funConst);
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
bool WarpBuilder::build_SpreadCall(BytecodeLocation loc) {
bool constructing = false;
CallInfo callInfo(alloc(), constructing, loc.resultIsPopped());
callInfo.initForSpreadCall(current);
// The argument must be an array object. Add an infallible MUnbox if needed,
// but ensure it's not loop hoisted before the branch in the bytecode guarding
// that it's not undefined.
MOZ_ASSERT(callInfo.argc() == 1);
callInfo.setArg(0, unboxObjectInfallible(callInfo.getArg(0), IsMovable::No));
if (auto* cacheIRSnapshot = getOpSnapshot<WarpCacheIR>(loc)) {
return transpileCall(loc, cacheIRSnapshot, &callInfo);
}
bool needsThisCheck = false;
MInstruction* call = makeSpreadCall(callInfo, needsThisCheck);
if (!call) {
return false;
}
call->setBailoutKind(BailoutKind::TooManyArguments);
current->add(call);
current->push(call);
return resumeAfter(call, loc);
}
bool WarpBuilder::build_SpreadNew(BytecodeLocation loc) {
bool constructing = true;
CallInfo callInfo(alloc(), constructing, loc.resultIsPopped());
callInfo.initForSpreadCall(current);
// See build_SpreadCall.
MOZ_ASSERT(callInfo.argc() == 1);
callInfo.setArg(0, unboxObjectInfallible(callInfo.getArg(0), IsMovable::No));
if (auto* cacheIRSnapshot = getOpSnapshot<WarpCacheIR>(loc)) {
return transpileCall(loc, cacheIRSnapshot, &callInfo);
}
buildCreateThis(callInfo);
bool needsThisCheck = true;
MInstruction* call = makeSpreadCall(callInfo, needsThisCheck);
if (!call) {
return false;
}
call->setBailoutKind(BailoutKind::TooManyArguments);
current->add(call);
current->push(call);
return resumeAfter(call, loc);
}
bool WarpBuilder::build_SpreadSuperCall(BytecodeLocation loc) {
return build_SpreadNew(loc);
}
bool WarpBuilder::build_OptimizeSpreadCall(BytecodeLocation loc) {
MDefinition* value = current->pop();
return buildIC(loc, CacheKind::OptimizeSpreadCall, {value});
}
bool WarpBuilder::build_Debugger(BytecodeLocation loc) {
// The |debugger;| statement will bail out to Baseline if the realm is a
// debuggee realm with an onDebuggerStatement hook.
MDebugger* debugger = MDebugger::New(alloc());
current->add(debugger);
return resumeAfter(debugger, loc);
}
bool WarpBuilder::build_TableSwitch(BytecodeLocation loc) {
int32_t low = loc.getTableSwitchLow();
int32_t high = loc.getTableSwitchHigh();
size_t numCases = high - low + 1;
MDefinition* input = current->pop();
MTableSwitch* tableswitch = MTableSwitch::New(alloc(), input, low, high);
current->end(tableswitch);
// Table mapping from target bytecode offset to MTableSwitch successor index.
// This prevents adding multiple predecessor/successor edges to the same
// target block, which isn't valid in MIR.
using TargetToSuccessorMap =
InlineMap<uint32_t, uint32_t, 8, DefaultHasher<uint32_t>,
SystemAllocPolicy>;
TargetToSuccessorMap targetToSuccessor;
// Create |default| edge.
{
BytecodeLocation defaultLoc = loc.getTableSwitchDefaultTarget();
uint32_t defaultOffset = defaultLoc.bytecodeToOffset(script_);
size_t index;
if (!tableswitch->addDefault(nullptr, &index)) {
return false;
}
if (!addPendingEdge(defaultLoc, current, index)) {
return false;
}
if (!targetToSuccessor.put(defaultOffset, index)) {
return false;
}
}
// Add all cases.
for (size_t i = 0; i < numCases; i++) {
BytecodeLocation caseLoc = loc.getTableSwitchCaseTarget(script_, i);
uint32_t caseOffset = caseLoc.bytecodeToOffset(script_);
size_t index;
if (auto p = targetToSuccessor.lookupForAdd(caseOffset)) {
index = p->value();
} else {
if (!tableswitch->addSuccessor(nullptr, &index)) {
return false;
}
if (!addPendingEdge(caseLoc, current, index)) {
return false;
}
if (!targetToSuccessor.add(p, caseOffset, index)) {
return false;
}
}
if (!tableswitch->addCase(index)) {
return false;
}
}
setTerminatedBlock();
return true;
}
bool WarpBuilder::build_Rest(BytecodeLocation loc) {
Shape* shape = getOpSnapshot<WarpRest>(loc)->shape();
// NOTE: Keep this code in sync with |ArgumentsReplacer|.
if (inlineCallInfo()) {
// If we are inlining, we know the actual arguments.
unsigned numActuals = inlineCallInfo()->argc();
unsigned numFormals = info().nargs() - 1;
unsigned numRest = numActuals > numFormals ? numActuals - numFormals : 0;
// TODO: support pre-tenuring.
gc::Heap heap = gc::Heap::Default;
// Allocate an array of the correct size.
MInstruction* newArray;
if (gc::CanUseFixedElementsForArray(numRest)) {
auto* shapeConstant = MConstant::NewShape(alloc(), shape);
current->add(shapeConstant);
newArray = MNewArrayObject::New(alloc(), shapeConstant, numRest, heap);
} else {
MConstant* templateConst = constant(NullValue());
newArray = MNewArray::NewVM(alloc(), numRest, templateConst, heap);
}
current->add(newArray);
current->push(newArray);
if (numRest == 0) {
// No more updating to do.
return true;
}
MElements* elements = MElements::New(alloc(), newArray);
current->add(elements);
// Unroll the argument copy loop. We don't need to do any bounds or hole
// checking here.
for (uint32_t i = numFormals; i < numActuals; i++) {
if (!alloc().ensureBallast()) {
return false;
}
MConstant* index = MConstant::NewInt32(alloc(), i - numFormals);
current->add(index);
MDefinition* arg = inlineCallInfo()->argv()[i];
MStoreElement* store =
MStoreElement::NewNoPreBarrier(alloc(), elements, index, arg,
/* needsHoleCheck = */ false);
current->add(store);
}
// Update the initialized length for all the (necessarily non-hole)
// elements added.
MSetInitializedLength* initLength =
MSetInitializedLength::New(alloc(), elements, numRest,
/* needsPreBarrier = */ false);
current->add(initLength);
return true;
}
MArgumentsLength* numActuals = MArgumentsLength::New(alloc());
current->add(numActuals);
// Pass in the number of actual arguments, the number of formals (not
// including the rest parameter slot itself), and the shape.
unsigned numFormals = info().nargs() - 1;
MRest* rest = MRest::New(alloc(), numActuals, numFormals, shape);
current->add(rest);
current->push(rest);
return true;
}
bool WarpBuilder::build_Try(BytecodeLocation loc) {
graph().setHasTryBlock();
MBasicBlock* pred = current;
if (!startNewBlock(pred, loc.next())) {
return false;
}
pred->end(MGoto::New(alloc(), current));
return true;
}
bool WarpBuilder::build_Finally(BytecodeLocation loc) {
MOZ_ASSERT(graph().hasTryBlock());
return true;
}
bool WarpBuilder::build_Exception(BytecodeLocation) {
MOZ_CRASH("Unreachable because we skip catch-blocks");
}
bool WarpBuilder::build_ExceptionAndStack(BytecodeLocation) {
MOZ_CRASH("Unreachable because we skip catch-blocks");
}
bool WarpBuilder::build_CreateSuppressedError(BytecodeLocation) {
MOZ_CRASH("Unreachable because we skip catch-blocks");
}
bool WarpBuilder::build_AsyncReject(BytecodeLocation) {
MOZ_CRASH("Unreachable because we skip catch-blocks");
}
bool WarpBuilder::build_Throw(BytecodeLocation loc) {
MDefinition* def = current->pop();
MThrow* ins = MThrow::New(alloc(), def);
current->add(ins);
if (!resumeAfter(ins, loc)) {
return false;
}
// Terminate the block.
current->end(MUnreachable::New(alloc()));
setTerminatedBlock();
return true;
}
bool WarpBuilder::build_ThrowWithStack(BytecodeLocation loc) {
MDefinition* stack = current->pop();
MDefinition* value = current->pop();
auto* ins = MThrowWithStack::New(alloc(), value, stack);
current->add(ins);
if (!resumeAfter(ins, loc)) {
return false;
}
// Terminate the block.
current->end(MUnreachable::New(alloc()));
setTerminatedBlock();
return true;
}
bool WarpBuilder::build_ThrowSetConst(BytecodeLocation loc) {
auto* ins = MThrowRuntimeLexicalError::New(alloc(), JSMSG_BAD_CONST_ASSIGN);
current->add(ins);
if (!resumeAfter(ins, loc)) {
return false;
}
// Terminate the block.
current->end(MUnreachable::New(alloc()));
setTerminatedBlock();
return true;
}
bool WarpBuilder::build_ThrowMsg(BytecodeLocation loc) {
auto* ins = MThrowMsg::New(alloc(), loc.throwMsgKind());
current->add(ins);
if (!resumeAfter(ins, loc)) {
return false;
}
// Terminate the block.
current->end(MUnreachable::New(alloc()));
setTerminatedBlock();
return true;
}
static bool CanTruncateToInt32(MIRType type) {
// Don't allow Strings because `MTruncateToInt32` can't directly truncate
// them to Int32. And it's probably not much more efficient if Strings are
// first boxed and then truncated through `LValueTruncateToInt32`.
return IsTypeRepresentableAsDouble(type) || IsNullOrUndefined(type) ||
type == MIRType::Boolean;
}
// Try to emit a bitwise instruction if both operands can be truncated to Int32.
static MInstruction* TryBitwise(TempAllocator& alloc, JSOp jsop,
MDefinition* lhs, MDefinition* rhs) {
switch (jsop) {
case JSOp::BitOr:
case JSOp::BitXor:
case JSOp::BitAnd:
case JSOp::Lsh:
case JSOp::Rsh:
break;
case JSOp::Ursh:
// Ursh is complicated because we don't know which MUrsh to use, so we
// don't yet optimize it here.
return nullptr;
default:
return nullptr;
}
// Both operands must be convertible to Int32 using truncation.
if (!CanTruncateToInt32(lhs->type()) || !CanTruncateToInt32(rhs->type())) {
return nullptr;
}
// It's valid to check the operand types even for loop phis, because loop
// phis are always typed as `MIRType::Value` at this point.
MOZ_ASSERT(lhs->type() != MIRType::Value && rhs->type() != MIRType::Value);
switch (jsop) {
case JSOp::BitOr:
return MBitOr::New(alloc, lhs, rhs, MIRType::Int32);
case JSOp::BitXor:
return MBitXor::New(alloc, lhs, rhs, MIRType::Int32);
case JSOp::BitAnd:
return MBitAnd::New(alloc, lhs, rhs, MIRType::Int32);
case JSOp::Lsh:
return MLsh::New(alloc, lhs, rhs, MIRType::Int32);
case JSOp::Rsh:
return MRsh::New(alloc, lhs, rhs, MIRType::Int32);
default:
break;
}
MOZ_CRASH("unexpected jsop");
}
// Try to emit a compare instruction if both operands are unboxed.
static MInstruction* TryCompare(TempAllocator& alloc, JSOp jsop,
MDefinition* lhs, MDefinition* rhs) {
MOZ_ASSERT(IsEqualityOp(jsop) || IsRelationalOp(jsop));
// Both operands must be unboxed.
//
// It's valid to check the operand types even for loop phis, because loop
// phis are always typed as `MIRType::Value` at this point.
if (lhs->type() == MIRType::Value || rhs->type() == MIRType::Value) {
return nullptr;
}
// Prefer MCompare if both operands have the same type.
if (lhs->type() == rhs->type()) {
MCompare::CompareType compareType;
switch (lhs->type()) {
case MIRType::Int32:
compareType = MCompare::Compare_Int32;
break;
case MIRType::Float32:
compareType = MCompare::Compare_Float32;
break;
case MIRType::Double:
compareType = MCompare::Compare_Double;
break;
case MIRType::String:
compareType = MCompare::Compare_String;
break;
case MIRType::BigInt:
compareType = MCompare::Compare_BigInt;
break;
default:
return nullptr;
}
return MCompare::New(alloc, lhs, rhs, jsop, compareType);
}
// Use MCompare if both operands are Numbers.
if (IsTypeRepresentableAsDouble(lhs->type()) &&
IsTypeRepresentableAsDouble(rhs->type())) {
return MCompare::New(alloc, lhs, rhs, jsop, MCompare::Compare_Double);
}
return nullptr;
}
// Try to emit a strict-constant-compare instruction.
static MInstruction* TryStrictConstantCompare(TempAllocator& alloc, JSOp jsop,
MDefinition* lhs,
MDefinition* rhs) {
// Need strict equality comparison.
if (!IsStrictEqualityOp(jsop)) {
return nullptr;
}
// At least one operand must be a constant.
if (!lhs->isConstant() && !rhs->isConstant()) {
return nullptr;
}
auto strictCompare = [&](MConstant* cst,
MDefinition* operand) -> MInstruction* {
// Strict equality comparison against an Int32 constant, but the constant
// is too large for the StrictConstantEq/Ne byte code.
if (cst->type() == MIRType::Int32) {
cst->setImplicitlyUsedUnchecked();
return MStrictConstantCompareInt32::New(alloc, operand, cst->toInt32(),
jsop);
}
// Strict equality comparison against a String constant.
if (cst->type() == MIRType::String) {
cst->setImplicitlyUsedUnchecked();
return MStrictConstantCompareString::New(alloc, operand, cst->toString(),
jsop);
}
// Strict equality comparison against an Object constant.
if (cst->type() == MIRType::Object) {
cst->setImplicitlyUsedUnchecked();
return MStrictConstantCompareObject::New(alloc, operand, &cst->toObject(),
jsop);
}
// Strict equality comparison against Undefined.
if (cst->type() == MIRType::Undefined) {
return MCompare::New(alloc, operand, cst, jsop,
MCompare::Compare_Undefined);
}
// Strict equality comparison against Null.
if (cst->type() == MIRType::Null) {
return MCompare::New(alloc, operand, cst, jsop, MCompare::Compare_Null);
}
// No optimization possible.
return nullptr;
};
if (lhs->isConstant()) {
if (auto* ins = strictCompare(lhs->toConstant(), rhs)) {
return ins;
}
}
if (rhs->isConstant()) {
if (auto* ins = strictCompare(rhs->toConstant(), lhs)) {
return ins;
}
}
return nullptr;
}
bool WarpBuilder::buildIC(BytecodeLocation loc, CacheKind kind,
std::initializer_list<MDefinition*> inputs) {
MOZ_ASSERT(loc.opHasIC());
mozilla::DebugOnly<size_t> numInputs = inputs.size();
MOZ_ASSERT(numInputs == NumInputsForCacheKind(kind));
const WarpCacheIRBase* cacheIRSnapshot = getOpSnapshot<WarpCacheIR>(loc);
if (!cacheIRSnapshot) {
cacheIRSnapshot = getOpSnapshot<WarpCacheIRWithShapeList>(loc);
if (!cacheIRSnapshot) {
cacheIRSnapshot = getOpSnapshot<WarpCacheIRWithShapeListAndOffsets>(loc);
}
}
if (cacheIRSnapshot) {
return TranspileCacheIRToMIR(this, loc, cacheIRSnapshot, inputs);
}
if (getOpSnapshot<WarpBailout>(loc)) {
for (MDefinition* input : inputs) {
input->setImplicitlyUsedUnchecked();
}
return buildBailoutForColdIC(loc, kind);
}
if (const auto* inliningSnapshot = getOpSnapshot<WarpInlinedCall>(loc)) {
// The CallInfo will be initialized by the transpiler.
bool ignoresRval = BytecodeIsPopped(loc.toRawBytecode());
CallInfo callInfo(alloc(), /*constructing =*/false, ignoresRval);
callInfo.markAsInlined();
if (!TranspileCacheIRToMIR(this, loc, inliningSnapshot->cacheIRSnapshot(),
inputs, &callInfo)) {
return false;
}
return buildInlinedCall(loc, inliningSnapshot, callInfo);
}
// Work around std::initializer_list not defining operator[].
auto getInput = [&](size_t index) -> MDefinition* {
MOZ_ASSERT(index < numInputs);
return inputs.begin()[index];
};
switch (kind) {
case CacheKind::UnaryArith: {
MOZ_ASSERT(numInputs == 1);
auto* ins = MUnaryCache::New(alloc(), getInput(0));
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
case CacheKind::ToPropertyKey: {
MOZ_ASSERT(numInputs == 1);
auto* ins = MToPropertyKeyCache::New(alloc(), getInput(0));
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
case CacheKind::BinaryArith: {
MOZ_ASSERT(numInputs == 2);
auto* lhs = getInput(0);
auto* rhs = getInput(1);
// If both operands are unboxed, we may be able to emit a more optimized
// instruction than just MBinaryCache.
//
// We perform this optimization early instead of in MBinaryCache::foldsTo
// to avoid creating resume points which may keep some operands alive.
if (auto* ins = TryBitwise(alloc(), loc.getOp(), lhs, rhs)) {
current->add(ins);
current->push(ins);
return true;
}
auto* ins = MBinaryCache::New(alloc(), lhs, rhs, MIRType::Value);
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
case CacheKind::Compare: {
MOZ_ASSERT(numInputs == 2);
auto* lhs = getInput(0);
auto* rhs = getInput(1);
// If both operands are unboxed, we may be able to emit a more optimized
// instruction than just MBinaryCache.
if (auto* ins = TryCompare(alloc(), loc.getOp(), lhs, rhs)) {
current->add(ins);
current->push(ins);
return true;
}
// If one operand is a constant, we may be able to create strict constant
// compare instructions.
if (auto* ins =
TryStrictConstantCompare(alloc(), loc.getOp(), lhs, rhs)) {
current->add(ins);
current->push(ins);
return true;
}
auto* ins = MBinaryCache::New(alloc(), lhs, rhs, MIRType::Boolean);
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
case CacheKind::In: {
MOZ_ASSERT(numInputs == 2);
auto* ins = MInCache::New(alloc(), getInput(0), getInput(1));
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
case CacheKind::HasOwn: {
MOZ_ASSERT(numInputs == 2);
// Note: the MHasOwnCache constructor takes obj/id instead of id/obj.
auto* ins = MHasOwnCache::New(alloc(), getInput(1), getInput(0));
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
case CacheKind::CheckPrivateField: {
MOZ_ASSERT(numInputs == 2);
auto* ins =
MCheckPrivateFieldCache::New(alloc(), getInput(0), getInput(1));
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
case CacheKind::InstanceOf: {
MOZ_ASSERT(numInputs == 2);
auto* ins = MInstanceOfCache::New(alloc(), getInput(0), getInput(1));
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
case CacheKind::BindName: {
MOZ_ASSERT(numInputs == 1);
auto* ins = MBindNameCache::New(alloc(), getInput(0));
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
case CacheKind::GetIterator: {
MOZ_ASSERT(numInputs == 1);
auto* ins = MGetIteratorCache::New(alloc(), getInput(0));
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
case CacheKind::GetName: {
MOZ_ASSERT(numInputs == 1);
auto* ins = MGetNameCache::New(alloc(), getInput(0));
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
case CacheKind::GetProp: {
MOZ_ASSERT(numInputs == 1);
PropertyName* name = loc.getPropertyName(script_);
MConstant* id = constant(StringValue(name));
MDefinition* val = getInput(0);
auto* ins = MGetPropertyCache::New(alloc(), val, id);
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
case CacheKind::GetElem: {
MOZ_ASSERT(numInputs == 2);
MDefinition* val = getInput(0);
auto* ins = MGetPropertyCache::New(alloc(), val, getInput(1));
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
case CacheKind::SetProp: {
MOZ_ASSERT(numInputs == 2);
PropertyName* name = loc.getPropertyName(script_);
MConstant* id = constant(StringValue(name));
bool strict = loc.isStrictSetOp();
auto* ins =
MSetPropertyCache::New(alloc(), getInput(0), id, getInput(1), strict);
current->add(ins);
return resumeAfter(ins, loc);
}
case CacheKind::SetElem: {
MOZ_ASSERT(numInputs == 3);
bool strict = loc.isStrictSetOp();
auto* ins = MSetPropertyCache::New(alloc(), getInput(0), getInput(1),
getInput(2), strict);
current->add(ins);
return resumeAfter(ins, loc);
}
case CacheKind::GetPropSuper: {
MOZ_ASSERT(numInputs == 2);
PropertyName* name = loc.getPropertyName(script_);
MConstant* id = constant(StringValue(name));
auto* ins =
MGetPropSuperCache::New(alloc(), getInput(0), getInput(1), id);
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
case CacheKind::GetElemSuper: {
MOZ_ASSERT(numInputs == 3);
// Note: CacheIR expects obj/id/receiver but MGetPropSuperCache takes
// obj/receiver/id so swap the last two inputs.
auto* ins = MGetPropSuperCache::New(alloc(), getInput(0), getInput(2),
getInput(1));
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
case CacheKind::OptimizeSpreadCall: {
MOZ_ASSERT(numInputs == 1);
auto* ins = MOptimizeSpreadCallCache::New(alloc(), getInput(0));
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
case CacheKind::TypeOf: {
// Note: Warp does not have a TypeOf IC, it just inlines the operation.
MOZ_ASSERT(numInputs == 1);
auto* typeOf = MTypeOf::New(alloc(), getInput(0));
current->add(typeOf);
auto* ins = MTypeOfName::New(alloc(), typeOf);
current->add(ins);
current->push(ins);
return true;
}
case CacheKind::TypeOfEq: {
MOZ_ASSERT(numInputs == 1);
auto operand = loc.getTypeofEqOperand();
JSType type = operand.type();
JSOp compareOp = operand.compareOp();
auto* typeOf = MTypeOf::New(alloc(), getInput(0));
current->add(typeOf);
auto* typeInt = MConstant::NewInt32(alloc(), type);
current->add(typeInt);
auto* ins = MCompare::New(alloc(), typeOf, typeInt, compareOp,
MCompare::Compare_Int32);
current->add(ins);
current->push(ins);
return true;
}
case CacheKind::NewObject: {
auto* templateConst = constant(NullValue());
MNewObject* ins = MNewObject::NewVM(
alloc(), templateConst, gc::Heap::Default, MNewObject::ObjectLiteral);
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
case CacheKind::NewArray: {
uint32_t length = loc.getNewArrayLength();
MConstant* templateConst = constant(NullValue());
MNewArray* ins =
MNewArray::NewVM(alloc(), length, templateConst, gc::Heap::Default);
current->add(ins);
current->push(ins);
return true;
}
case CacheKind::CloseIter: {
MOZ_ASSERT(numInputs == 1);
static_assert(sizeof(CompletionKind) == sizeof(uint8_t));
CompletionKind kind = loc.getCompletionKind();
auto* ins = MCloseIterCache::New(alloc(), getInput(0), uint8_t(kind));
current->add(ins);
return resumeAfter(ins, loc);
}
case CacheKind::OptimizeGetIterator: {
MOZ_ASSERT(numInputs == 1);
auto* ins = MOptimizeGetIteratorCache::New(alloc(), getInput(0));
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
case CacheKind::Lambda: {
MDefinition* env = current->environmentChain();
JSFunction* fun = loc.getFunction(script_);
MConstant* funConst = constant(ObjectValue(*fun));
auto* ins = MLambda::New(alloc(), env, funConst, gc::Heap::Default);
current->add(ins);
current->push(ins);
return resumeAfter(ins, loc);
}
case CacheKind::LazyConstant:
case CacheKind::ToBool:
case CacheKind::Call:
case CacheKind::GetImport:
// We're currently not using an IC or transpiling CacheIR for these kinds.
MOZ_CRASH("Unexpected kind");
}
return true;
}
bool WarpBuilder::buildBailoutForColdIC(BytecodeLocation loc, CacheKind kind) {
MOZ_ASSERT(loc.opHasIC());
MBail* bail = MBail::New(alloc(), BailoutKind::FirstExecution);
current->add(bail);
current->setAlwaysBails();
MIRType resultType;
switch (kind) {
case CacheKind::UnaryArith:
case CacheKind::BinaryArith:
case CacheKind::GetName:
case CacheKind::GetProp:
case CacheKind::GetElem:
case CacheKind::GetPropSuper:
case CacheKind::GetElemSuper:
case CacheKind::GetImport:
case CacheKind::LazyConstant:
case CacheKind::Call:
case CacheKind::ToPropertyKey:
case CacheKind::OptimizeSpreadCall:
resultType = MIRType::Value;
break;
case CacheKind::BindName:
case CacheKind::GetIterator:
case CacheKind::NewArray:
case CacheKind::NewObject:
case CacheKind::Lambda:
resultType = MIRType::Object;
break;
case CacheKind::TypeOf:
resultType = MIRType::String;
break;
case CacheKind::ToBool:
case CacheKind::Compare:
case CacheKind::In:
case CacheKind::HasOwn:
case CacheKind::CheckPrivateField:
case CacheKind::InstanceOf:
case CacheKind::OptimizeGetIterator:
case CacheKind::TypeOfEq:
resultType = MIRType::Boolean;
break;
case CacheKind::SetProp:
case CacheKind::SetElem:
case CacheKind::CloseIter:
return true; // No result.
}
auto* ins = MUnreachableResult::New(alloc(), resultType);
current->add(ins);
current->push(ins);
return true;
}
class MOZ_RAII AutoAccumulateReturns {
MIRGraph& graph_;
MIRGraphReturns* prev_;
public:
AutoAccumulateReturns(MIRGraph& graph, MIRGraphReturns& returns)
: graph_(graph) {
prev_ = graph_.returnAccumulator();
graph_.setReturnAccumulator(&returns);
}
~AutoAccumulateReturns() { graph_.setReturnAccumulator(prev_); }
};
bool WarpBuilder::buildInlinedCall(BytecodeLocation loc,
const WarpInlinedCall* inlineSnapshot,
CallInfo& callInfo) {
jsbytecode* pc = loc.toRawBytecode();
if (callInfo.isSetter()) {
// build_SetProp pushes the rhs argument onto the stack. Remove it
// in preparation for pushCallStack.
current->pop();
}
callInfo.setImplicitlyUsedUnchecked();
// Capture formals in the outer resume point.
if (!callInfo.pushCallStack(current)) {
return false;
}
MResumePoint* outerResumePoint =
MResumePoint::New(alloc(), current, pc, callInfo.inliningResumeMode());
if (!outerResumePoint) {
return false;
}
current->setOuterResumePoint(outerResumePoint);
// Pop formals again, except leave |callee| on stack for duration of call.
callInfo.popCallStack(current);
current->push(callInfo.callee());
// Build the graph.
CompileInfo* calleeCompileInfo = inlineSnapshot->info();
MIRGraphReturns returns(alloc());
AutoAccumulateReturns aar(graph(), returns);
WarpBuilder inlineBuilder(this, inlineSnapshot->scriptSnapshot(),
*calleeCompileInfo, &callInfo, outerResumePoint);
if (!inlineBuilder.buildInline()) {
// Note: Inlining only aborts on OOM. If inlining would fail for
// any other reason, we detect it in advance and don't inline.
return false;
}
// We mark scripts as uninlineable in BytecodeAnalysis if we cannot
// reach a return statement (without going through a catch/finally).
MOZ_ASSERT(!returns.empty());
// Create return block
BytecodeLocation postCall = loc.next();
MBasicBlock* prev = current;
if (!startNewEntryBlock(prev->stackDepth(), postCall)) {
return false;
}
// Restore previous value of callerResumePoint.
current->setCallerResumePoint(callerResumePoint());
current->inheritSlots(prev);
// Pop |callee|.
current->pop();
// Accumulate return values.
MDefinition* returnValue =
patchInlinedReturns(calleeCompileInfo, callInfo, returns, current);
if (!returnValue) {
return false;
}
current->push(returnValue);
// Initialize entry slots
if (!current->initEntrySlots(alloc())) {
return false;
}
return true;
}
MDefinition* WarpBuilder::patchInlinedReturns(CompileInfo* calleeCompileInfo,
CallInfo& callInfo,