Source code
Revision control
Copy as Markdown
Other Tools
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: set ts=8 sts=2 et sw=2 tw=80:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
#include "jit/BaselineCodeGen.h"
#include "mozilla/Casting.h"
#include "gc/GC.h"
#include "jit/BaselineIC.h"
#include "jit/BaselineJIT.h"
#include "jit/CacheIRCompiler.h"
#include "jit/CacheIRGenerator.h"
#include "jit/CalleeToken.h"
#include "jit/FixedList.h"
#include "jit/IonOptimizationLevels.h"
#include "jit/JitcodeMap.h"
#include "jit/JitFrames.h"
#include "jit/JitRuntime.h"
#include "jit/JitSpewer.h"
#include "jit/Linker.h"
#include "jit/PerfSpewer.h"
#include "jit/SharedICHelpers.h"
#include "jit/TemplateObject.h"
#include "jit/TrialInlining.h"
#include "jit/VMFunctions.h"
#include "js/friend/ErrorMessages.h" // JSMSG_*
#include "js/UniquePtr.h"
#include "vm/AsyncFunction.h"
#include "vm/AsyncIteration.h"
#include "vm/BuiltinObjectKind.h"
#include "vm/EnvironmentObject.h"
#include "vm/FunctionFlags.h" // js::FunctionFlags
#include "vm/Interpreter.h"
#include "vm/JSFunction.h"
#include "vm/Time.h"
#ifdef MOZ_VTUNE
# include "vtune/VTuneWrapper.h"
#endif
#include "debugger/DebugAPI-inl.h"
#include "jit/BaselineFrameInfo-inl.h"
#include "jit/JitHints-inl.h"
#include "jit/JitScript-inl.h"
#include "jit/MacroAssembler-inl.h"
#include "jit/SharedICHelpers-inl.h"
#include "jit/TemplateObject-inl.h"
#include "jit/VMFunctionList-inl.h"
#include "vm/Interpreter-inl.h"
#include "vm/JSScript-inl.h"
using namespace js;
using namespace js::jit;
using JS::TraceKind;
using mozilla::AssertedCast;
using mozilla::Maybe;
namespace js {
class PlainObject;
namespace jit {
BaselineCompilerHandler::BaselineCompilerHandler(JSContext* cx,
MacroAssembler& masm,
TempAllocator& alloc,
JSScript* script)
: frame_(script, masm),
alloc_(alloc),
analysis_(alloc, script),
#ifdef DEBUG
masm_(masm),
#endif
script_(script),
pc_(script->code()),
icEntryIndex_(0),
compileDebugInstrumentation_(script->isDebuggee()),
ionCompileable_(IsIonEnabled(cx) && CanIonCompileScript(cx, script)) {
}
BaselineInterpreterHandler::BaselineInterpreterHandler(JSContext* cx,
MacroAssembler& masm)
: frame_(masm) {}
template <typename Handler>
template <typename... HandlerArgs>
BaselineCodeGen<Handler>::BaselineCodeGen(JSContext* cx, TempAllocator& alloc,
HandlerArgs&&... args)
: handler(cx, masm, std::forward<HandlerArgs>(args)...),
cx(cx),
masm(cx, alloc),
frame(handler.frame()) {}
BaselineCompiler::BaselineCompiler(JSContext* cx, TempAllocator& alloc,
JSScript* script)
: BaselineCodeGen(cx, alloc, /* HandlerArgs = */ alloc, script) {
#ifdef JS_CODEGEN_NONE
MOZ_CRASH();
#endif
}
BaselineInterpreterGenerator::BaselineInterpreterGenerator(JSContext* cx,
TempAllocator& alloc)
: BaselineCodeGen(cx, alloc /* no handlerArgs */) {}
bool BaselineCompilerHandler::init(JSContext* cx) {
if (!analysis_.init(alloc_)) {
return false;
}
uint32_t len = script_->length();
if (!labels_.init(alloc_, len)) {
return false;
}
for (size_t i = 0; i < len; i++) {
new (&labels_[i]) Label();
}
if (!frame_.init(alloc_)) {
return false;
}
return true;
}
bool BaselineCompiler::init() {
if (!handler.init(cx)) {
return false;
}
return true;
}
bool BaselineCompilerHandler::recordCallRetAddr(JSContext* cx,
RetAddrEntry::Kind kind,
uint32_t retOffset) {
uint32_t pcOffset = script_->pcToOffset(pc_);
// Entries must be sorted by pcOffset for binary search to work.
// See BaselineScript::retAddrEntryFromPCOffset.
MOZ_ASSERT_IF(!retAddrEntries_.empty(),
retAddrEntries_.back().pcOffset() <= pcOffset);
// Similarly, entries must be sorted by return offset and this offset must be
// unique. See BaselineScript::retAddrEntryFromReturnOffset.
MOZ_ASSERT_IF(!retAddrEntries_.empty() && !masm_.oom(),
retAddrEntries_.back().returnOffset().offset() < retOffset);
if (!retAddrEntries_.emplaceBack(pcOffset, kind, CodeOffset(retOffset))) {
ReportOutOfMemory(cx);
return false;
}
return true;
}
bool BaselineInterpreterHandler::recordCallRetAddr(JSContext* cx,
RetAddrEntry::Kind kind,
uint32_t retOffset) {
switch (kind) {
case RetAddrEntry::Kind::DebugPrologue:
MOZ_ASSERT(callVMOffsets_.debugPrologueOffset == 0,
"expected single DebugPrologue call");
callVMOffsets_.debugPrologueOffset = retOffset;
break;
case RetAddrEntry::Kind::DebugEpilogue:
MOZ_ASSERT(callVMOffsets_.debugEpilogueOffset == 0,
"expected single DebugEpilogue call");
callVMOffsets_.debugEpilogueOffset = retOffset;
break;
case RetAddrEntry::Kind::DebugAfterYield:
MOZ_ASSERT(callVMOffsets_.debugAfterYieldOffset == 0,
"expected single DebugAfterYield call");
callVMOffsets_.debugAfterYieldOffset = retOffset;
break;
default:
break;
}
return true;
}
bool BaselineInterpreterHandler::addDebugInstrumentationOffset(
JSContext* cx, CodeOffset offset) {
if (!debugInstrumentationOffsets_.append(offset.offset())) {
ReportOutOfMemory(cx);
return false;
}
return true;
}
MethodStatus BaselineCompiler::compile() {
AutoCreatedBy acb(masm, "BaselineCompiler::compile");
Rooted<JSScript*> script(cx, handler.script());
JitSpew(JitSpew_BaselineScripts, "Baseline compiling script %s:%u:%u (%p)",
script->filename(), script->lineno(),
script->column().oneOriginValue(), script.get());
JitSpew(JitSpew_Codegen, "# Emitting baseline code for script %s:%u:%u",
script->filename(), script->lineno(),
script->column().oneOriginValue());
AutoIncrementalTimer timer(cx->realm()->timers.baselineCompileTime);
AutoKeepJitScripts keepJitScript(cx);
if (!script->ensureHasJitScript(cx, keepJitScript)) {
return Method_Error;
}
// When code coverage is enabled, we have to create the ScriptCounts if they
// do not exist.
if (!script->hasScriptCounts() && cx->realm()->collectCoverageForDebug()) {
if (!script->initScriptCounts(cx)) {
return Method_Error;
}
}
if (!JitOptions.disableJitHints &&
cx->runtime()->jitRuntime()->hasJitHintsMap()) {
JitHintsMap* jitHints = cx->runtime()->jitRuntime()->getJitHintsMap();
jitHints->setEagerBaselineHint(script);
}
// Suppress GC during compilation.
gc::AutoSuppressGC suppressGC(cx);
if (!script->jitScript()->ensureHasCachedBaselineJitData(cx, script)) {
return Method_Error;
}
MOZ_ASSERT(!script->hasBaselineScript());
perfSpewer_.recordOffset(masm, "Prologue");
if (!emitPrologue()) {
return Method_Error;
}
MethodStatus status = emitBody();
if (status != Method_Compiled) {
return status;
}
perfSpewer_.recordOffset(masm, "Epilogue");
if (!emitEpilogue()) {
return Method_Error;
}
perfSpewer_.recordOffset(masm, "OOLPostBarrierSlot");
if (!emitOutOfLinePostBarrierSlot()) {
return Method_Error;
}
AutoCreatedBy acb2(masm, "exception_tail");
Linker linker(masm);
if (masm.oom()) {
ReportOutOfMemory(cx);
return Method_Error;
}
JitCode* code = linker.newCode(cx, CodeKind::Baseline);
if (!code) {
return Method_Error;
}
UniquePtr<BaselineScript> baselineScript(
BaselineScript::New(
cx, warmUpCheckPrologueOffset_.offset(),
profilerEnterFrameToggleOffset_.offset(),
profilerExitFrameToggleOffset_.offset(),
handler.retAddrEntries().length(), handler.osrEntries().length(),
debugTrapEntries_.length(), script->resumeOffsets().size()),
JS::DeletePolicy<BaselineScript>(cx->runtime()));
if (!baselineScript) {
return Method_Error;
}
baselineScript->setMethod(code);
JitSpew(JitSpew_BaselineScripts,
"Created BaselineScript %p (raw %p) for %s:%u:%u",
(void*)baselineScript.get(), (void*)code->raw(), script->filename(),
script->lineno(), script->column().oneOriginValue());
baselineScript->copyRetAddrEntries(handler.retAddrEntries().begin());
baselineScript->copyOSREntries(handler.osrEntries().begin());
baselineScript->copyDebugTrapEntries(debugTrapEntries_.begin());
// If profiler instrumentation is enabled, toggle instrumentation on.
if (cx->runtime()->jitRuntime()->isProfilerInstrumentationEnabled(
cx->runtime())) {
baselineScript->toggleProfilerInstrumentation(true);
}
// Compute native resume addresses for the script's resume offsets.
baselineScript->computeResumeNativeOffsets(script, resumeOffsetEntries_);
if (compileDebugInstrumentation()) {
baselineScript->setHasDebugInstrumentation();
}
// Always register a native => bytecode mapping entry, since profiler can be
// turned on with baseline jitcode on stack, and baseline jitcode cannot be
// invalidated.
{
JitSpew(JitSpew_Profiling,
"Added JitcodeGlobalEntry for baseline script %s:%u:%u (%p)",
script->filename(), script->lineno(),
script->column().oneOriginValue(), baselineScript.get());
// Generate profiling string.
UniqueChars str = GeckoProfilerRuntime::allocProfileString(cx, script);
if (!str) {
return Method_Error;
}
auto entry = MakeJitcodeGlobalEntry<BaselineEntry>(
cx, code, code->raw(), code->rawEnd(), script, std::move(str));
if (!entry) {
return Method_Error;
}
JitcodeGlobalTable* globalTable =
cx->runtime()->jitRuntime()->getJitcodeGlobalTable();
if (!globalTable->addEntry(std::move(entry))) {
ReportOutOfMemory(cx);
return Method_Error;
}
// Mark the jitcode as having a bytecode map.
code->setHasBytecodeMap();
}
script->jitScript()->setBaselineScript(script, baselineScript.release());
perfSpewer_.saveProfile(cx, script, code);
#ifdef MOZ_VTUNE
vtune::MarkScript(code, script, "baseline");
#endif
return Method_Compiled;
}
// On most platforms we use a dedicated bytecode PC register to avoid many
// dependent loads and stores for sequences of simple bytecode ops. This
// register must be saved/restored around VM and IC calls.
//
// On 32-bit x86 we don't have enough registers for this (because R0-R2 require
// 6 registers) so there we always store the pc on the frame.
static constexpr bool HasInterpreterPCReg() {
return InterpreterPCReg != InvalidReg;
}
static Register LoadBytecodePC(MacroAssembler& masm, Register scratch) {
if (HasInterpreterPCReg()) {
return InterpreterPCReg;
}
Address pcAddr(FramePointer, BaselineFrame::reverseOffsetOfInterpreterPC());
masm.loadPtr(pcAddr, scratch);
return scratch;
}
static void LoadInt8Operand(MacroAssembler& masm, Register dest) {
Register pc = LoadBytecodePC(masm, dest);
masm.load8SignExtend(Address(pc, sizeof(jsbytecode)), dest);
}
static void LoadUint8Operand(MacroAssembler& masm, Register dest) {
Register pc = LoadBytecodePC(masm, dest);
masm.load8ZeroExtend(Address(pc, sizeof(jsbytecode)), dest);
}
static void LoadUint16Operand(MacroAssembler& masm, Register dest) {
Register pc = LoadBytecodePC(masm, dest);
masm.load16ZeroExtend(Address(pc, sizeof(jsbytecode)), dest);
}
static void LoadInt32Operand(MacroAssembler& masm, Register dest) {
Register pc = LoadBytecodePC(masm, dest);
masm.load32(Address(pc, sizeof(jsbytecode)), dest);
}
static void LoadInt32OperandSignExtendToPtr(MacroAssembler& masm, Register pc,
Register dest) {
masm.load32SignExtendToPtr(Address(pc, sizeof(jsbytecode)), dest);
}
static void LoadUint24Operand(MacroAssembler& masm, size_t offset,
Register dest) {
// Load the opcode and operand, then left shift to discard the opcode.
Register pc = LoadBytecodePC(masm, dest);
masm.load32(Address(pc, offset), dest);
masm.rshift32(Imm32(8), dest);
}
static void LoadInlineValueOperand(MacroAssembler& masm, ValueOperand dest) {
// Note: the Value might be unaligned but as above we rely on all our
// platforms having appropriate support for unaligned accesses (except for
// floating point instructions on ARM).
Register pc = LoadBytecodePC(masm, dest.scratchReg());
masm.loadUnalignedValue(Address(pc, sizeof(jsbytecode)), dest);
}
template <>
void BaselineCompilerCodeGen::loadScript(Register dest) {
masm.movePtr(ImmGCPtr(handler.script()), dest);
}
template <>
void BaselineInterpreterCodeGen::loadScript(Register dest) {
masm.loadPtr(frame.addressOfInterpreterScript(), dest);
}
template <>
void BaselineCompilerCodeGen::saveInterpreterPCReg() {}
template <>
void BaselineInterpreterCodeGen::saveInterpreterPCReg() {
if (HasInterpreterPCReg()) {
masm.storePtr(InterpreterPCReg, frame.addressOfInterpreterPC());
}
}
template <>
void BaselineCompilerCodeGen::restoreInterpreterPCReg() {}
template <>
void BaselineInterpreterCodeGen::restoreInterpreterPCReg() {
if (HasInterpreterPCReg()) {
masm.loadPtr(frame.addressOfInterpreterPC(), InterpreterPCReg);
}
}
template <>
void BaselineCompilerCodeGen::emitInitializeLocals() {
// Initialize all locals to |undefined|. Lexical bindings are temporal
// dead zoned in bytecode.
size_t n = frame.nlocals();
if (n == 0) {
return;
}
// Use R0 to minimize code size. If the number of locals to push is <
// LOOP_UNROLL_FACTOR, then the initialization pushes are emitted directly
// and inline. Otherwise, they're emitted in a partially unrolled loop.
static const size_t LOOP_UNROLL_FACTOR = 4;
size_t toPushExtra = n % LOOP_UNROLL_FACTOR;
masm.moveValue(UndefinedValue(), R0);
// Handle any extra pushes left over by the optional unrolled loop below.
for (size_t i = 0; i < toPushExtra; i++) {
masm.pushValue(R0);
}
// Partially unrolled loop of pushes.
if (n >= LOOP_UNROLL_FACTOR) {
size_t toPush = n - toPushExtra;
MOZ_ASSERT(toPush % LOOP_UNROLL_FACTOR == 0);
MOZ_ASSERT(toPush >= LOOP_UNROLL_FACTOR);
masm.move32(Imm32(toPush), R1.scratchReg());
// Emit unrolled loop with 4 pushes per iteration.
Label pushLoop;
masm.bind(&pushLoop);
for (size_t i = 0; i < LOOP_UNROLL_FACTOR; i++) {
masm.pushValue(R0);
}
masm.branchSub32(Assembler::NonZero, Imm32(LOOP_UNROLL_FACTOR),
R1.scratchReg(), &pushLoop);
}
}
template <>
void BaselineInterpreterCodeGen::emitInitializeLocals() {
// Push |undefined| for all locals.
Register scratch = R0.scratchReg();
loadScript(scratch);
masm.loadPtr(Address(scratch, JSScript::offsetOfSharedData()), scratch);
masm.loadPtr(Address(scratch, SharedImmutableScriptData::offsetOfISD()),
scratch);
masm.load32(Address(scratch, ImmutableScriptData::offsetOfNfixed()), scratch);
Label top, done;
masm.branchTest32(Assembler::Zero, scratch, scratch, &done);
masm.bind(&top);
{
masm.pushValue(UndefinedValue());
masm.branchSub32(Assembler::NonZero, Imm32(1), scratch, &top);
}
masm.bind(&done);
}
// On input:
// R2.scratchReg() contains object being written to.
// Called with the baseline stack synced, except for R0 which is preserved.
// All other registers are usable as scratch.
// This calls:
// void PostWriteBarrier(JSRuntime* rt, JSObject* obj);
template <typename Handler>
bool BaselineCodeGen<Handler>::emitOutOfLinePostBarrierSlot() {
AutoCreatedBy acb(masm,
"BaselineCodeGen<Handler>::emitOutOfLinePostBarrierSlot");
if (!postBarrierSlot_.used()) {
return true;
}
masm.bind(&postBarrierSlot_);
#ifdef JS_USE_LINK_REGISTER
masm.pushReturnAddress();
#endif
Register objReg = R2.scratchReg();
// Check one element cache to avoid VM call.
Label skipBarrier;
auto* lastCellAddr = cx->runtime()->gc.addressOfLastBufferedWholeCell();
masm.branchPtr(Assembler::Equal, AbsoluteAddress(lastCellAddr), objReg,
&skipBarrier);
saveInterpreterPCReg();
AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
MOZ_ASSERT(!regs.has(FramePointer));
regs.take(R0);
regs.take(objReg);
Register scratch = regs.takeAny();
masm.pushValue(R0);
using Fn = void (*)(JSRuntime* rt, js::gc::Cell* cell);
masm.setupUnalignedABICall(scratch);
masm.movePtr(ImmPtr(cx->runtime()), scratch);
masm.passABIArg(scratch);
masm.passABIArg(objReg);
masm.callWithABI<Fn, PostWriteBarrier>();
restoreInterpreterPCReg();
masm.popValue(R0);
masm.bind(&skipBarrier);
masm.ret();
return true;
}
// Scan the a cache IR stub's fields and create an allocation site for any that
// refer to the catch-all unknown allocation site. This will be the case for
// stubs created when running in the interpreter. This happens on transition to
// baseline.
static bool CreateAllocSitesForCacheIRStub(JSScript* script, uint32_t pcOffset,
ICCacheIRStub* stub) {
const CacheIRStubInfo* stubInfo = stub->stubInfo();
uint8_t* stubData = stub->stubDataStart();
ICScript* icScript = script->jitScript()->icScript();
uint32_t field = 0;
size_t offset = 0;
while (true) {
StubField::Type fieldType = stubInfo->fieldType(field);
if (fieldType == StubField::Type::Limit) {
break;
}
if (fieldType == StubField::Type::AllocSite) {
gc::AllocSite* site =
stubInfo->getPtrStubField<ICCacheIRStub, gc::AllocSite>(stub, offset);
if (site->kind() == gc::AllocSite::Kind::Unknown) {
gc::AllocSite* newSite =
icScript->getOrCreateAllocSite(script, pcOffset);
if (!newSite) {
return false;
}
stubInfo->replaceStubRawWord(stubData, offset, uintptr_t(site),
uintptr_t(newSite));
}
}
field++;
offset += StubField::sizeInBytes(fieldType);
}
return true;
}
static void CreateAllocSitesForICChain(JSScript* script, uint32_t pcOffset,
uint32_t entryIndex) {
JitScript* jitScript = script->jitScript();
ICStub* stub = jitScript->icEntry(entryIndex).firstStub();
while (!stub->isFallback()) {
if (!CreateAllocSitesForCacheIRStub(script, pcOffset,
stub->toCacheIRStub())) {
// This is an optimization and safe to skip if we hit OOM or per-zone
// limit.
return;
}
stub = stub->toCacheIRStub()->next();
}
}
template <>
bool BaselineCompilerCodeGen::emitNextIC() {
AutoCreatedBy acb(masm, "emitNextIC");
// Emit a call to an IC stored in JitScript. Calls to this must match the
// ICEntry order in JitScript: first the non-op IC entries for |this| and
// formal arguments, then the for-op IC entries for JOF_IC ops.
JSScript* script = handler.script();
uint32_t pcOffset = script->pcToOffset(handler.pc());
// We don't use every ICEntry and we can skip unreachable ops, so we have
// to loop until we find an ICEntry for the current pc.
const ICFallbackStub* stub;
uint32_t entryIndex;
do {
stub = script->jitScript()->fallbackStub(handler.icEntryIndex());
entryIndex = handler.icEntryIndex();
handler.moveToNextICEntry();
} while (stub->pcOffset() < pcOffset);
MOZ_ASSERT(stub->pcOffset() == pcOffset);
MOZ_ASSERT(BytecodeOpHasIC(JSOp(*handler.pc())));
if (BytecodeOpCanHaveAllocSite(JSOp(*handler.pc()))) {
CreateAllocSitesForICChain(script, pcOffset, entryIndex);
}
// Load stub pointer into ICStubReg.
masm.loadPtr(frame.addressOfICScript(), ICStubReg);
size_t firstStubOffset = ICScript::offsetOfFirstStub(entryIndex);
masm.loadPtr(Address(ICStubReg, firstStubOffset), ICStubReg);
CodeOffset returnOffset;
EmitCallIC(masm, &returnOffset);
RetAddrEntry::Kind kind = RetAddrEntry::Kind::IC;
if (!handler.retAddrEntries().emplaceBack(pcOffset, kind, returnOffset)) {
ReportOutOfMemory(cx);
return false;
}
return true;
}
template <>
bool BaselineInterpreterCodeGen::emitNextIC() {
saveInterpreterPCReg();
masm.loadPtr(frame.addressOfInterpreterICEntry(), ICStubReg);
masm.loadPtr(Address(ICStubReg, ICEntry::offsetOfFirstStub()), ICStubReg);
masm.call(Address(ICStubReg, ICStub::offsetOfStubCode()));
uint32_t returnOffset = masm.currentOffset();
restoreInterpreterPCReg();
// If this is an IC for a bytecode op where Ion may inline scripts, we need to
// record the return offset for Ion bailouts.
if (handler.currentOp()) {
JSOp op = *handler.currentOp();
MOZ_ASSERT(BytecodeOpHasIC(op));
if (IsIonInlinableOp(op)) {
if (!handler.icReturnOffsets().emplaceBack(returnOffset, op)) {
return false;
}
}
}
return true;
}
template <>
void BaselineCompilerCodeGen::computeFrameSize(Register dest) {
MOZ_ASSERT(!inCall_, "must not be called in the middle of a VM call");
masm.move32(Imm32(frame.frameSize()), dest);
}
template <>
void BaselineInterpreterCodeGen::computeFrameSize(Register dest) {
// dest := FramePointer - StackPointer.
MOZ_ASSERT(!inCall_, "must not be called in the middle of a VM call");
masm.mov(FramePointer, dest);
masm.subStackPtrFrom(dest);
}
template <typename Handler>
void BaselineCodeGen<Handler>::prepareVMCall() {
pushedBeforeCall_ = masm.framePushed();
#ifdef DEBUG
inCall_ = true;
#endif
// Ensure everything is synced.
frame.syncStack(0);
}
template <>
void BaselineCompilerCodeGen::storeFrameSizeAndPushDescriptor(
uint32_t argSize, Register scratch) {
#ifdef DEBUG
masm.store32(Imm32(frame.frameSize()), frame.addressOfDebugFrameSize());
#endif
masm.pushFrameDescriptor(FrameType::BaselineJS);
}
template <>
void BaselineInterpreterCodeGen::storeFrameSizeAndPushDescriptor(
uint32_t argSize, Register scratch) {
#ifdef DEBUG
// Store the frame size without VMFunction arguments in debug builds.
// scratch := FramePointer - StackPointer - argSize.
masm.mov(FramePointer, scratch);
masm.subStackPtrFrom(scratch);
masm.sub32(Imm32(argSize), scratch);
masm.store32(scratch, frame.addressOfDebugFrameSize());
#endif
masm.pushFrameDescriptor(FrameType::BaselineJS);
}
static uint32_t GetVMFunctionArgSize(const VMFunctionData& fun) {
return fun.explicitStackSlots() * sizeof(void*);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::callVMInternal(VMFunctionId id,
RetAddrEntry::Kind kind,
CallVMPhase phase) {
#ifdef DEBUG
// Assert prepareVMCall() has been called.
MOZ_ASSERT(inCall_);
inCall_ = false;
#endif
TrampolinePtr code = cx->runtime()->jitRuntime()->getVMWrapper(id);
const VMFunctionData& fun = GetVMFunction(id);
uint32_t argSize = GetVMFunctionArgSize(fun);
// Assert all arguments were pushed.
MOZ_ASSERT(masm.framePushed() - pushedBeforeCall_ == argSize);
saveInterpreterPCReg();
if (phase == CallVMPhase::AfterPushingLocals) {
storeFrameSizeAndPushDescriptor(argSize, R0.scratchReg());
} else {
MOZ_ASSERT(phase == CallVMPhase::BeforePushingLocals);
#ifdef DEBUG
uint32_t frameBaseSize = BaselineFrame::frameSizeForNumValueSlots(0);
masm.store32(Imm32(frameBaseSize), frame.addressOfDebugFrameSize());
#endif
masm.pushFrameDescriptor(FrameType::BaselineJS);
}
// Perform the call.
masm.call(code);
uint32_t callOffset = masm.currentOffset();
// Pop arguments from framePushed.
masm.implicitPop(argSize);
restoreInterpreterPCReg();
return handler.recordCallRetAddr(cx, kind, callOffset);
}
template <typename Handler>
template <typename Fn, Fn fn>
bool BaselineCodeGen<Handler>::callVM(RetAddrEntry::Kind kind,
CallVMPhase phase) {
VMFunctionId fnId = VMFunctionToId<Fn, fn>::id;
return callVMInternal(fnId, kind, phase);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emitStackCheck() {
Label skipCall;
if (handler.mustIncludeSlotsInStackCheck()) {
// Subtract the size of script->nslots() first.
Register scratch = R1.scratchReg();
masm.moveStackPtrTo(scratch);
subtractScriptSlotsSize(scratch, R2.scratchReg());
masm.branchPtr(Assembler::BelowOrEqual,
AbsoluteAddress(cx->addressOfJitStackLimit()), scratch,
&skipCall);
} else {
masm.branchStackPtrRhs(Assembler::BelowOrEqual,
AbsoluteAddress(cx->addressOfJitStackLimit()),
&skipCall);
}
prepareVMCall();
masm.loadBaselineFramePtr(FramePointer, R1.scratchReg());
pushArg(R1.scratchReg());
const CallVMPhase phase = CallVMPhase::BeforePushingLocals;
const RetAddrEntry::Kind kind = RetAddrEntry::Kind::StackCheck;
using Fn = bool (*)(JSContext*, BaselineFrame*);
if (!callVM<Fn, CheckOverRecursedBaseline>(kind, phase)) {
return false;
}
masm.bind(&skipCall);
return true;
}
static void EmitCallFrameIsDebuggeeCheck(MacroAssembler& masm) {
using Fn = void (*)(BaselineFrame* frame);
masm.setupUnalignedABICall(R0.scratchReg());
masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
masm.passABIArg(R0.scratchReg());
masm.callWithABI<Fn, FrameIsDebuggeeCheck>();
}
template <>
bool BaselineCompilerCodeGen::emitIsDebuggeeCheck() {
if (handler.compileDebugInstrumentation()) {
EmitCallFrameIsDebuggeeCheck(masm);
}
return true;
}
template <>
bool BaselineInterpreterCodeGen::emitIsDebuggeeCheck() {
// Use a toggled jump to call FrameIsDebuggeeCheck only if the debugger is
// enabled.
//
// call. Consider moving the callWithABI out-of-line.
Label skipCheck;
CodeOffset toggleOffset = masm.toggledJump(&skipCheck);
{
saveInterpreterPCReg();
EmitCallFrameIsDebuggeeCheck(masm);
restoreInterpreterPCReg();
}
masm.bind(&skipCheck);
return handler.addDebugInstrumentationOffset(cx, toggleOffset);
}
static void MaybeIncrementCodeCoverageCounter(MacroAssembler& masm,
JSScript* script,
jsbytecode* pc) {
if (!script->hasScriptCounts()) {
return;
}
PCCounts* counts = script->maybeGetPCCounts(pc);
uint64_t* counterAddr = &counts->numExec();
masm.inc64(AbsoluteAddress(counterAddr));
}
template <>
bool BaselineCompilerCodeGen::emitHandleCodeCoverageAtPrologue() {
// If the main instruction is not a jump target, then we emit the
// corresponding code coverage counter.
JSScript* script = handler.script();
jsbytecode* main = script->main();
if (!BytecodeIsJumpTarget(JSOp(*main))) {
MaybeIncrementCodeCoverageCounter(masm, script, main);
}
return true;
}
template <>
bool BaselineInterpreterCodeGen::emitHandleCodeCoverageAtPrologue() {
Label skipCoverage;
CodeOffset toggleOffset = masm.toggledJump(&skipCoverage);
masm.call(handler.codeCoverageAtPrologueLabel());
masm.bind(&skipCoverage);
return handler.codeCoverageOffsets().append(toggleOffset.offset());
}
template <>
void BaselineCompilerCodeGen::subtractScriptSlotsSize(Register reg,
Register scratch) {
uint32_t slotsSize = handler.script()->nslots() * sizeof(Value);
masm.subPtr(Imm32(slotsSize), reg);
}
template <>
void BaselineInterpreterCodeGen::subtractScriptSlotsSize(Register reg,
Register scratch) {
// reg = reg - script->nslots() * sizeof(Value)
MOZ_ASSERT(reg != scratch);
loadScript(scratch);
masm.loadPtr(Address(scratch, JSScript::offsetOfSharedData()), scratch);
masm.loadPtr(Address(scratch, SharedImmutableScriptData::offsetOfISD()),
scratch);
masm.load32(Address(scratch, ImmutableScriptData::offsetOfNslots()), scratch);
static_assert(sizeof(Value) == 8,
"shift by 3 below assumes Value is 8 bytes");
masm.lshiftPtr(Imm32(3), scratch);
masm.subPtr(scratch, reg);
}
template <>
void BaselineCompilerCodeGen::loadGlobalLexicalEnvironment(Register dest) {
MOZ_ASSERT(!handler.script()->hasNonSyntacticScope());
masm.movePtr(ImmGCPtr(&cx->global()->lexicalEnvironment()), dest);
}
template <>
void BaselineInterpreterCodeGen::loadGlobalLexicalEnvironment(Register dest) {
masm.loadGlobalObjectData(dest);
masm.loadPtr(Address(dest, GlobalObjectData::offsetOfLexicalEnvironment()),
dest);
}
template <>
void BaselineCompilerCodeGen::pushGlobalLexicalEnvironmentValue(
ValueOperand scratch) {
frame.push(ObjectValue(cx->global()->lexicalEnvironment()));
}
template <>
void BaselineInterpreterCodeGen::pushGlobalLexicalEnvironmentValue(
ValueOperand scratch) {
loadGlobalLexicalEnvironment(scratch.scratchReg());
masm.tagValue(JSVAL_TYPE_OBJECT, scratch.scratchReg(), scratch);
frame.push(scratch);
}
template <>
void BaselineCompilerCodeGen::loadGlobalThisValue(ValueOperand dest) {
JSObject* thisObj = cx->global()->lexicalEnvironment().thisObject();
masm.moveValue(ObjectValue(*thisObj), dest);
}
template <>
void BaselineInterpreterCodeGen::loadGlobalThisValue(ValueOperand dest) {
Register scratch = dest.scratchReg();
loadGlobalLexicalEnvironment(scratch);
static constexpr size_t SlotOffset =
GlobalLexicalEnvironmentObject::offsetOfThisValueSlot();
masm.loadValue(Address(scratch, SlotOffset), dest);
}
template <>
void BaselineCompilerCodeGen::pushScriptArg() {
pushArg(ImmGCPtr(handler.script()));
}
template <>
void BaselineInterpreterCodeGen::pushScriptArg() {
pushArg(frame.addressOfInterpreterScript());
}
template <>
void BaselineCompilerCodeGen::pushBytecodePCArg() {
pushArg(ImmPtr(handler.pc()));
}
template <>
void BaselineInterpreterCodeGen::pushBytecodePCArg() {
if (HasInterpreterPCReg()) {
pushArg(InterpreterPCReg);
} else {
pushArg(frame.addressOfInterpreterPC());
}
}
static gc::Cell* GetScriptGCThing(JSScript* script, jsbytecode* pc,
ScriptGCThingType type) {
switch (type) {
case ScriptGCThingType::Atom:
return script->getAtom(pc);
case ScriptGCThingType::String:
return script->getString(pc);
case ScriptGCThingType::RegExp:
return script->getRegExp(pc);
case ScriptGCThingType::Object:
return script->getObject(pc);
case ScriptGCThingType::Function:
return script->getFunction(pc);
case ScriptGCThingType::Scope:
return script->getScope(pc);
case ScriptGCThingType::BigInt:
return script->getBigInt(pc);
}
MOZ_CRASH("Unexpected GCThing type");
}
template <>
void BaselineCompilerCodeGen::loadScriptGCThing(ScriptGCThingType type,
Register dest,
Register scratch) {
gc::Cell* thing = GetScriptGCThing(handler.script(), handler.pc(), type);
masm.movePtr(ImmGCPtr(thing), dest);
}
template <>
void BaselineInterpreterCodeGen::loadScriptGCThing(ScriptGCThingType type,
Register dest,
Register scratch) {
MOZ_ASSERT(dest != scratch);
// Load the index in |scratch|.
LoadInt32Operand(masm, scratch);
// Load the GCCellPtr.
loadScript(dest);
masm.loadPtr(Address(dest, JSScript::offsetOfPrivateData()), dest);
masm.loadPtr(BaseIndex(dest, scratch, ScalePointer,
PrivateScriptData::offsetOfGCThings()),
dest);
// Clear the tag bits.
switch (type) {
case ScriptGCThingType::Atom:
case ScriptGCThingType::String:
// Use xorPtr with a 32-bit immediate because it's more efficient than
// andPtr on 64-bit.
static_assert(uintptr_t(TraceKind::String) == 2,
"Unexpected tag bits for string GCCellPtr");
masm.xorPtr(Imm32(2), dest);
break;
case ScriptGCThingType::RegExp:
case ScriptGCThingType::Object:
case ScriptGCThingType::Function:
// No-op because GCCellPtr tag bits are zero for objects.
static_assert(uintptr_t(TraceKind::Object) == 0,
"Unexpected tag bits for object GCCellPtr");
break;
case ScriptGCThingType::BigInt:
// Use xorPtr with a 32-bit immediate because it's more efficient than
// andPtr on 64-bit.
static_assert(uintptr_t(TraceKind::BigInt) == 1,
"Unexpected tag bits for BigInt GCCellPtr");
masm.xorPtr(Imm32(1), dest);
break;
case ScriptGCThingType::Scope:
// Use xorPtr with a 32-bit immediate because it's more efficient than
// andPtr on 64-bit.
static_assert(uintptr_t(TraceKind::Scope) >= JS::OutOfLineTraceKindMask,
"Expected Scopes to have OutOfLineTraceKindMask tag");
masm.xorPtr(Imm32(JS::OutOfLineTraceKindMask), dest);
break;
}
#ifdef DEBUG
// Assert low bits are not set.
Label ok;
masm.branchTestPtr(Assembler::Zero, dest, Imm32(0b111), &ok);
masm.assumeUnreachable("GC pointer with tag bits set");
masm.bind(&ok);
#endif
}
template <>
void BaselineCompilerCodeGen::pushScriptGCThingArg(ScriptGCThingType type,
Register scratch1,
Register scratch2) {
gc::Cell* thing = GetScriptGCThing(handler.script(), handler.pc(), type);
pushArg(ImmGCPtr(thing));
}
template <>
void BaselineInterpreterCodeGen::pushScriptGCThingArg(ScriptGCThingType type,
Register scratch1,
Register scratch2) {
loadScriptGCThing(type, scratch1, scratch2);
pushArg(scratch1);
}
template <typename Handler>
void BaselineCodeGen<Handler>::pushScriptNameArg(Register scratch1,
Register scratch2) {
pushScriptGCThingArg(ScriptGCThingType::Atom, scratch1, scratch2);
}
template <>
void BaselineCompilerCodeGen::pushUint8BytecodeOperandArg(Register) {
MOZ_ASSERT(JOF_OPTYPE(JSOp(*handler.pc())) == JOF_UINT8);
pushArg(Imm32(GET_UINT8(handler.pc())));
}
template <>
void BaselineInterpreterCodeGen::pushUint8BytecodeOperandArg(Register scratch) {
LoadUint8Operand(masm, scratch);
pushArg(scratch);
}
template <>
void BaselineCompilerCodeGen::pushUint16BytecodeOperandArg(Register) {
MOZ_ASSERT(JOF_OPTYPE(JSOp(*handler.pc())) == JOF_UINT16);
pushArg(Imm32(GET_UINT16(handler.pc())));
}
template <>
void BaselineInterpreterCodeGen::pushUint16BytecodeOperandArg(
Register scratch) {
LoadUint16Operand(masm, scratch);
pushArg(scratch);
}
template <>
void BaselineCompilerCodeGen::loadInt32LengthBytecodeOperand(Register dest) {
uint32_t length = GET_UINT32(handler.pc());
MOZ_ASSERT(length <= INT32_MAX,
"the bytecode emitter must fail to compile code that would "
"produce a length exceeding int32_t range");
masm.move32(Imm32(AssertedCast<int32_t>(length)), dest);
}
template <>
void BaselineInterpreterCodeGen::loadInt32LengthBytecodeOperand(Register dest) {
LoadInt32Operand(masm, dest);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emitDebugPrologue() {
auto ifDebuggee = [this]() {
// Load pointer to BaselineFrame in R0.
masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
prepareVMCall();
pushArg(R0.scratchReg());
const RetAddrEntry::Kind kind = RetAddrEntry::Kind::DebugPrologue;
using Fn = bool (*)(JSContext*, BaselineFrame*);
if (!callVM<Fn, jit::DebugPrologue>(kind)) {
return false;
}
return true;
};
return emitDebugInstrumentation(ifDebuggee);
}
template <>
void BaselineCompilerCodeGen::emitInitFrameFields(Register nonFunctionEnv) {
Register scratch = R0.scratchReg();
Register scratch2 = R2.scratchReg();
MOZ_ASSERT(nonFunctionEnv != scratch && nonFunctionEnv != scratch2);
masm.store32(Imm32(0), frame.addressOfFlags());
if (handler.function()) {
masm.loadFunctionFromCalleeToken(frame.addressOfCalleeToken(), scratch);
masm.unboxObject(Address(scratch, JSFunction::offsetOfEnvironment()),
scratch);
masm.storePtr(scratch, frame.addressOfEnvironmentChain());
} else {
masm.storePtr(nonFunctionEnv, frame.addressOfEnvironmentChain());
}
// If cx->inlinedICScript contains an inlined ICScript (passed from
// the caller), take that ICScript and store it in the frame, then
// overwrite cx->inlinedICScript with nullptr.
Label notInlined, done;
masm.movePtr(ImmPtr(cx->addressOfInlinedICScript()), scratch);
Address inlinedAddr(scratch, 0);
masm.branchPtr(Assembler::Equal, inlinedAddr, ImmWord(0), ¬Inlined);
masm.loadPtr(inlinedAddr, scratch2);
masm.storePtr(scratch2, frame.addressOfICScript());
masm.storePtr(ImmPtr(nullptr), inlinedAddr);
masm.jump(&done);
// Otherwise, store this script's default ICSCript in the frame.
masm.bind(¬Inlined);
masm.storePtr(ImmPtr(handler.script()->jitScript()->icScript()),
frame.addressOfICScript());
masm.bind(&done);
}
template <>
void BaselineInterpreterCodeGen::emitInitFrameFields(Register nonFunctionEnv) {
MOZ_ASSERT(nonFunctionEnv == R1.scratchReg(),
"Don't clobber nonFunctionEnv below");
// If we have a dedicated PC register we use it as scratch1 to avoid a
// register move below.
Register scratch1 =
HasInterpreterPCReg() ? InterpreterPCReg : R0.scratchReg();
Register scratch2 = R2.scratchReg();
masm.store32(Imm32(BaselineFrame::RUNNING_IN_INTERPRETER),
frame.addressOfFlags());
// Initialize interpreterScript.
Label notFunction, done;
masm.loadPtr(frame.addressOfCalleeToken(), scratch1);
masm.branchTestPtr(Assembler::NonZero, scratch1, Imm32(CalleeTokenScriptBit),
¬Function);
{
// CalleeToken_Function or CalleeToken_FunctionConstructing.
masm.andPtr(Imm32(uint32_t(CalleeTokenMask)), scratch1);
masm.unboxObject(Address(scratch1, JSFunction::offsetOfEnvironment()),
scratch2);
masm.storePtr(scratch2, frame.addressOfEnvironmentChain());
masm.loadPrivate(Address(scratch1, JSFunction::offsetOfJitInfoOrScript()),
scratch1);
masm.jump(&done);
}
masm.bind(¬Function);
{
// CalleeToken_Script.
masm.andPtr(Imm32(uint32_t(CalleeTokenMask)), scratch1);
masm.storePtr(nonFunctionEnv, frame.addressOfEnvironmentChain());
}
masm.bind(&done);
masm.storePtr(scratch1, frame.addressOfInterpreterScript());
// Initialize icScript and interpreterICEntry
masm.loadJitScript(scratch1, scratch2);
masm.computeEffectiveAddress(Address(scratch2, JitScript::offsetOfICScript()),
scratch2);
masm.storePtr(scratch2, frame.addressOfICScript());
masm.computeEffectiveAddress(Address(scratch2, ICScript::offsetOfICEntries()),
scratch2);
masm.storePtr(scratch2, frame.addressOfInterpreterICEntry());
// Initialize interpreter pc.
masm.loadPtr(Address(scratch1, JSScript::offsetOfSharedData()), scratch1);
masm.loadPtr(Address(scratch1, SharedImmutableScriptData::offsetOfISD()),
scratch1);
masm.addPtr(Imm32(ImmutableScriptData::offsetOfCode()), scratch1);
if (HasInterpreterPCReg()) {
MOZ_ASSERT(scratch1 == InterpreterPCReg,
"pc must be stored in the pc register");
} else {
masm.storePtr(scratch1, frame.addressOfInterpreterPC());
}
}
// Assert we don't need a post write barrier to write sourceObj to a slot of
// destObj. See comments in WarpBuilder::buildNamedLambdaEnv.
static void AssertCanElidePostWriteBarrier(MacroAssembler& masm,
Register destObj, Register sourceObj,
Register temp) {
#ifdef DEBUG
Label ok;
masm.branchPtrInNurseryChunk(Assembler::Equal, destObj, temp, &ok);
masm.branchPtrInNurseryChunk(Assembler::NotEqual, sourceObj, temp, &ok);
masm.assumeUnreachable("Unexpected missing post write barrier in Baseline");
masm.bind(&ok);
#endif
}
template <>
bool BaselineCompilerCodeGen::initEnvironmentChain() {
if (!handler.function()) {
return true;
}
if (!handler.script()->needsFunctionEnvironmentObjects()) {
return true;
}
// Allocate a NamedLambdaObject and/or a CallObject. If the function needs
// both, the NamedLambdaObject must enclose the CallObject. If one of the
// allocations fails, we perform the whole operation in C++.
JSObject* templateEnv = handler.script()->jitScript()->templateEnvironment();
MOZ_ASSERT(templateEnv);
CallObject* callObjectTemplate = nullptr;
if (handler.function()->needsCallObject()) {
callObjectTemplate = &templateEnv->as<CallObject>();
}
NamedLambdaObject* namedLambdaTemplate = nullptr;
if (handler.function()->needsNamedLambdaEnvironment()) {
if (callObjectTemplate) {
templateEnv = templateEnv->enclosingEnvironment();
}
namedLambdaTemplate = &templateEnv->as<NamedLambdaObject>();
}
MOZ_ASSERT(namedLambdaTemplate || callObjectTemplate);
AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
Register newEnv = regs.takeAny();
Register enclosingEnv = regs.takeAny();
Register callee = regs.takeAny();
Register temp = regs.takeAny();
Label fail;
masm.loadPtr(frame.addressOfEnvironmentChain(), enclosingEnv);
masm.loadFunctionFromCalleeToken(frame.addressOfCalleeToken(), callee);
// Allocate a NamedLambdaObject if needed.
if (namedLambdaTemplate) {
TemplateObject templateObject(namedLambdaTemplate);
masm.createGCObject(newEnv, temp, templateObject, gc::Heap::Default, &fail);
// Store enclosing environment.
Address enclosingSlot(newEnv,
NamedLambdaObject::offsetOfEnclosingEnvironment());
masm.storeValue(JSVAL_TYPE_OBJECT, enclosingEnv, enclosingSlot);
AssertCanElidePostWriteBarrier(masm, newEnv, enclosingEnv, temp);
// Store callee.
Address lambdaSlot(newEnv, NamedLambdaObject::offsetOfLambdaSlot());
masm.storeValue(JSVAL_TYPE_OBJECT, callee, lambdaSlot);
AssertCanElidePostWriteBarrier(masm, newEnv, callee, temp);
if (callObjectTemplate) {
masm.movePtr(newEnv, enclosingEnv);
}
}
// Allocate a CallObject if needed.
if (callObjectTemplate) {
TemplateObject templateObject(callObjectTemplate);
masm.createGCObject(newEnv, temp, templateObject, gc::Heap::Default, &fail);
// Store enclosing environment.
Address enclosingSlot(newEnv, CallObject::offsetOfEnclosingEnvironment());
masm.storeValue(JSVAL_TYPE_OBJECT, enclosingEnv, enclosingSlot);
AssertCanElidePostWriteBarrier(masm, newEnv, enclosingEnv, temp);
// Store callee.
Address calleeSlot(newEnv, CallObject::offsetOfCallee());
masm.storeValue(JSVAL_TYPE_OBJECT, callee, calleeSlot);
AssertCanElidePostWriteBarrier(masm, newEnv, callee, temp);
}
// Update the frame's environment chain and mark it initialized.
Label done;
masm.storePtr(newEnv, frame.addressOfEnvironmentChain());
masm.or32(Imm32(BaselineFrame::HAS_INITIAL_ENV), frame.addressOfFlags());
masm.jump(&done);
masm.bind(&fail);
prepareVMCall();
masm.loadBaselineFramePtr(FramePointer, temp);
pushArg(temp);
const CallVMPhase phase = CallVMPhase::BeforePushingLocals;
using Fn = bool (*)(JSContext*, BaselineFrame*);
if (!callVMNonOp<Fn, jit::InitFunctionEnvironmentObjects>(phase)) {
return false;
}
masm.bind(&done);
return true;
}
template <>
bool BaselineInterpreterCodeGen::initEnvironmentChain() {
// For function scripts, call InitFunctionEnvironmentObjects if needed. For
// non-function scripts this is a no-op.
Label done;
masm.branchTestPtr(Assembler::NonZero, frame.addressOfCalleeToken(),
Imm32(CalleeTokenScriptBit), &done);
{
auto initEnv = [this]() {
// Call into the VM to create the proper environment objects.
prepareVMCall();
masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
pushArg(R0.scratchReg());
const CallVMPhase phase = CallVMPhase::BeforePushingLocals;
using Fn = bool (*)(JSContext*, BaselineFrame*);
return callVMNonOp<Fn, jit::InitFunctionEnvironmentObjects>(phase);
};
if (!emitTestScriptFlag(
JSScript::ImmutableFlags::NeedsFunctionEnvironmentObjects, true,
initEnv, R2.scratchReg())) {
return false;
}
}
masm.bind(&done);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emitInterruptCheck() {
frame.syncStack(0);
Label done;
masm.branch32(Assembler::Equal, AbsoluteAddress(cx->addressOfInterruptBits()),
Imm32(0), &done);
prepareVMCall();
// Use a custom RetAddrEntry::Kind so DebugModeOSR can distinguish this call
// from other callVMs that might happen at this pc.
const RetAddrEntry::Kind kind = RetAddrEntry::Kind::InterruptCheck;
using Fn = bool (*)(JSContext*);
if (!callVM<Fn, InterruptCheck>(kind)) {
return false;
}
masm.bind(&done);
return true;
}
template <>
bool BaselineCompilerCodeGen::emitWarmUpCounterIncrement() {
frame.assertSyncedStack();
// Record native code offset for OSR from Baseline Interpreter into Baseline
// JIT code. This is right before the warm-up check in the Baseline JIT code,
// to make sure we can immediately enter Ion if the script is warm enough or
// if --ion-eager is used.
JSScript* script = handler.script();
jsbytecode* pc = handler.pc();
if (JSOp(*pc) == JSOp::LoopHead) {
uint32_t pcOffset = script->pcToOffset(pc);
uint32_t nativeOffset = masm.currentOffset();
if (!handler.osrEntries().emplaceBack(pcOffset, nativeOffset)) {
ReportOutOfMemory(cx);
return false;
}
}
// Emit no warm-up counter increments if Ion is not enabled or if the script
// will never be Ion-compileable.
if (!handler.maybeIonCompileable()) {
return true;
}
Register scriptReg = R2.scratchReg();
Register countReg = R0.scratchReg();
// Load the ICScript* in scriptReg.
masm.loadPtr(frame.addressOfICScript(), scriptReg);
// Bump warm-up counter.
Address warmUpCounterAddr(scriptReg, ICScript::offsetOfWarmUpCount());
masm.load32(warmUpCounterAddr, countReg);
masm.add32(Imm32(1), countReg);
masm.store32(countReg, warmUpCounterAddr);
if (!JitOptions.disableInlining) {
// Consider trial inlining.
// Note: unlike other warmup thresholds, where we try to enter a
// higher tier whenever we are higher than a given warmup count,
// trial inlining triggers once when reaching the threshold.
Label noTrialInlining;
masm.branch32(Assembler::NotEqual, countReg,
Imm32(JitOptions.trialInliningWarmUpThreshold),
&noTrialInlining);
prepareVMCall();
masm.PushBaselineFramePtr(FramePointer, R0.scratchReg());
using Fn = bool (*)(JSContext*, BaselineFrame*);
if (!callVMNonOp<Fn, DoTrialInlining>()) {
return false;
}
// Reload registers potentially clobbered by the call.
masm.loadPtr(frame.addressOfICScript(), scriptReg);
masm.load32(warmUpCounterAddr, countReg);
masm.bind(&noTrialInlining);
}
if (JSOp(*pc) == JSOp::LoopHead) {
// If this is a loop where we can't OSR (for example because it's inside a
// catch or finally block), increment the warmup counter but don't attempt
// OSR (Ion/Warp only compiles the try block).
if (!handler.analysis().info(pc).loopHeadCanOsr) {
return true;
}
}
Label done;
const OptimizationInfo* info =
IonOptimizations.get(OptimizationLevel::Normal);
uint32_t warmUpThreshold = info->compilerWarmUpThreshold(cx, script, pc);
masm.branch32(Assembler::LessThan, countReg, Imm32(warmUpThreshold), &done);
// Don't trigger Warp compilations from trial-inlined scripts.
Address depthAddr(scriptReg, ICScript::offsetOfDepth());
masm.branch32(Assembler::NotEqual, depthAddr, Imm32(0), &done);
// Load the IonScript* in scriptReg. We can load this from the ICScript*
// because it must be an outer ICScript embedded in the JitScript.
constexpr int32_t offset = -int32_t(JitScript::offsetOfICScript()) +
int32_t(JitScript::offsetOfIonScript());
masm.loadPtr(Address(scriptReg, offset), scriptReg);
// Do nothing if Ion is already compiling this script off-thread or if Ion has
// been disabled for this script.
masm.branchPtr(Assembler::Equal, scriptReg, ImmPtr(IonCompilingScriptPtr),
&done);
masm.branchPtr(Assembler::Equal, scriptReg, ImmPtr(IonDisabledScriptPtr),
&done);
// Try to compile and/or finish a compilation.
if (JSOp(*pc) == JSOp::LoopHead) {
// Try to OSR into Ion.
computeFrameSize(R0.scratchReg());
prepareVMCall();
pushBytecodePCArg();
pushArg(R0.scratchReg());
masm.PushBaselineFramePtr(FramePointer, R0.scratchReg());
using Fn = bool (*)(JSContext*, BaselineFrame*, uint32_t, jsbytecode*,
IonOsrTempData**);
if (!callVM<Fn, IonCompileScriptForBaselineOSR>()) {
return false;
}
// The return register holds the IonOsrTempData*. Perform OSR if it's not
// nullptr.
static_assert(ReturnReg != OsrFrameReg,
"Code below depends on osrDataReg != OsrFrameReg");
Register osrDataReg = ReturnReg;
masm.branchTestPtr(Assembler::Zero, osrDataReg, osrDataReg, &done);
// Success! Switch from Baseline JIT code to Ion JIT code.
// At this point, stack looks like:
//
// +-> [...Calling-Frame...]
// | [...Actual-Args/ThisV/ArgCount/Callee...]
// | [Descriptor]
// | [Return-Addr]
// +---[Saved-FramePtr]
// [...Baseline-Frame...]
#ifdef DEBUG
// Get a scratch register that's not osrDataReg or OsrFrameReg.
AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
MOZ_ASSERT(!regs.has(FramePointer));
regs.take(osrDataReg);
regs.take(OsrFrameReg);
Register scratchReg = regs.takeAny();
// If profiler instrumentation is on, ensure that lastProfilingFrame is
// the frame currently being OSR-ed
{
Label checkOk;
AbsoluteAddress addressOfEnabled(
cx->runtime()->geckoProfiler().addressOfEnabled());
masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0), &checkOk);
masm.loadPtr(AbsoluteAddress((void*)&cx->jitActivation), scratchReg);
masm.loadPtr(
Address(scratchReg, JitActivation::offsetOfLastProfilingFrame()),
scratchReg);
// It may be the case that we entered the baseline frame with
// profiling turned off on, then in a call within a loop (i.e. a
// callee frame), turn on profiling, then return to this frame,
// and then OSR with profiling turned on. In this case, allow for
// lastProfilingFrame to be null.
masm.branchPtr(Assembler::Equal, scratchReg, ImmWord(0), &checkOk);
masm.branchPtr(Assembler::Equal, FramePointer, scratchReg, &checkOk);
masm.assumeUnreachable("Baseline OSR lastProfilingFrame mismatch.");
masm.bind(&checkOk);
}
#endif
// Restore the stack pointer so that the saved frame pointer is on top of
// the stack.
masm.moveToStackPtr(FramePointer);
// Jump into Ion.
masm.loadPtr(Address(osrDataReg, IonOsrTempData::offsetOfBaselineFrame()),
OsrFrameReg);
masm.jump(Address(osrDataReg, IonOsrTempData::offsetOfJitCode()));
} else {
prepareVMCall();
masm.PushBaselineFramePtr(FramePointer, R0.scratchReg());
using Fn = bool (*)(JSContext*, BaselineFrame*);
if (!callVMNonOp<Fn, IonCompileScriptForBaselineAtEntry>()) {
return false;
}
}
masm.bind(&done);
return true;
}
template <>
bool BaselineInterpreterCodeGen::emitWarmUpCounterIncrement() {
Register scriptReg = R2.scratchReg();
Register countReg = R0.scratchReg();
// Load the JitScript* in scriptReg.
loadScript(scriptReg);
masm.loadJitScript(scriptReg, scriptReg);
// Bump warm-up counter.
Address warmUpCounterAddr(scriptReg, JitScript::offsetOfWarmUpCount());
masm.load32(warmUpCounterAddr, countReg);
masm.add32(Imm32(1), countReg);
masm.store32(countReg, warmUpCounterAddr);
// If the script is warm enough for Baseline compilation, call into the VM to
// compile it.
Label done;
masm.branch32(Assembler::BelowOrEqual, countReg,
Imm32(JitOptions.baselineJitWarmUpThreshold), &done);
masm.branchPtr(Assembler::Equal,
Address(scriptReg, JitScript::offsetOfBaselineScript()),
ImmPtr(BaselineDisabledScriptPtr), &done);
{
prepareVMCall();
masm.PushBaselineFramePtr(FramePointer, R0.scratchReg());
using Fn = bool (*)(JSContext*, BaselineFrame*, uint8_t**);
if (!callVM<Fn, BaselineCompileFromBaselineInterpreter>()) {
return false;
}
// If the function returned nullptr we either skipped compilation or were
// unable to compile the script. Continue running in the interpreter.
masm.branchTestPtr(Assembler::Zero, ReturnReg, ReturnReg, &done);
// Success! Switch from interpreter to JIT code by jumping to the
// corresponding code in the BaselineScript.
//
// This works because BaselineCompiler uses the same frame layout (stack is
// synced at OSR points) and BaselineCompileFromBaselineInterpreter has
// already cleared the RUNNING_IN_INTERPRETER flag for us.
// See BaselineFrame::prepareForBaselineInterpreterToJitOSR.
masm.jump(ReturnReg);
}
masm.bind(&done);
return true;
}
bool BaselineCompiler::emitDebugTrap() {
MOZ_ASSERT(compileDebugInstrumentation());
MOZ_ASSERT(frame.numUnsyncedSlots() == 0);
JSScript* script = handler.script();
bool enabled = DebugAPI::stepModeEnabled(script) ||
DebugAPI::hasBreakpointsAt(script, handler.pc());
// Emit patchable call to debug trap handler.
JitCode* handlerCode = cx->runtime()->jitRuntime()->debugTrapHandler(
cx, DebugTrapHandlerKind::Compiler);
if (!handlerCode) {
return false;
}
CodeOffset nativeOffset = masm.toggledCall(handlerCode, enabled);
uint32_t pcOffset = script->pcToOffset(handler.pc());
if (!debugTrapEntries_.emplaceBack(pcOffset, nativeOffset.offset())) {
ReportOutOfMemory(cx);
return false;
}
// Add a RetAddrEntry for the return offset -> pc mapping.
return handler.recordCallRetAddr(cx, RetAddrEntry::Kind::DebugTrap,
masm.currentOffset());
}
template <typename Handler>
void BaselineCodeGen<Handler>::emitProfilerEnterFrame() {
// Store stack position to lastProfilingFrame variable, guarded by a toggled
// jump. Starts off initially disabled.
Label noInstrument;
CodeOffset toggleOffset = masm.toggledJump(&noInstrument);
masm.profilerEnterFrame(FramePointer, R0.scratchReg());
masm.bind(&noInstrument);
// Store the start offset in the appropriate location.
MOZ_ASSERT(!profilerEnterFrameToggleOffset_.bound());
profilerEnterFrameToggleOffset_ = toggleOffset;
}
template <typename Handler>
void BaselineCodeGen<Handler>::emitProfilerExitFrame() {
// Store previous frame to lastProfilingFrame variable, guarded by a toggled
// jump. Starts off initially disabled.
Label noInstrument;
CodeOffset toggleOffset = masm.toggledJump(&noInstrument);
masm.profilerExitFrame();
masm.bind(&noInstrument);
// Store the start offset in the appropriate location.
MOZ_ASSERT(!profilerExitFrameToggleOffset_.bound());
profilerExitFrameToggleOffset_ = toggleOffset;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Nop() {
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_NopDestructuring() {
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_NopIsAssignOp() {
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_TryDestructuring() {
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Pop() {
frame.pop();
return true;
}
template <>
bool BaselineCompilerCodeGen::emit_PopN() {
frame.popn(GET_UINT16(handler.pc()));
return true;
}
template <>
bool BaselineInterpreterCodeGen::emit_PopN() {
LoadUint16Operand(masm, R0.scratchReg());
frame.popn(R0.scratchReg());
return true;
}
template <>
bool BaselineCompilerCodeGen::emit_DupAt() {
frame.syncStack(0);
// DupAt takes a value on the stack and re-pushes it on top. It's like
// GetLocal but it addresses from the top of the stack instead of from the
// stack frame.
int depth = -(GET_UINT24(handler.pc()) + 1);
masm.loadValue(frame.addressOfStackValue(depth), R0);
frame.push(R0);
return true;
}
template <>
bool BaselineInterpreterCodeGen::emit_DupAt() {
LoadUint24Operand(masm, 0, R0.scratchReg());
masm.loadValue(frame.addressOfStackValue(R0.scratchReg()), R0);
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Dup() {
// Keep top stack value in R0, sync the rest so that we can use R1. We use
// separate registers because every register can be used by at most one
// StackValue.
frame.popRegsAndSync(1);
masm.moveValue(R0, R1);
// inc/dec ops use Dup followed by Inc/Dec. Push R0 last to avoid a move.
frame.push(R1);
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Dup2() {
frame.syncStack(0);
masm.loadValue(frame.addressOfStackValue(-2), R0);
masm.loadValue(frame.addressOfStackValue(-1), R1);
frame.push(R0);
frame.push(R1);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Swap() {
// Keep top stack values in R0 and R1.
frame.popRegsAndSync(2);
frame.push(R1);
frame.push(R0);
return true;
}
template <>
bool BaselineCompilerCodeGen::emit_Pick() {
frame.syncStack(0);
// Pick takes a value on the stack and moves it to the top.
// For instance, pick 2:
// before: A B C D E
// after : A B D E C
// First, move value at -(amount + 1) into R0.
int32_t depth = -(GET_INT8(handler.pc()) + 1);
masm.loadValue(frame.addressOfStackValue(depth), R0);
// Move the other values down.
depth++;
for (; depth < 0; depth++) {
Address source = frame.addressOfStackValue(depth);
Address dest = frame.addressOfStackValue(depth - 1);
masm.loadValue(source, R1);
masm.storeValue(R1, dest);
}
// Push R0.
frame.pop();
frame.push(R0);
return true;
}
template <>
bool BaselineInterpreterCodeGen::emit_Pick() {
// First, move the value to move up into R0.
Register scratch = R2.scratchReg();
LoadUint8Operand(masm, scratch);
masm.loadValue(frame.addressOfStackValue(scratch), R0);
// Move the other values down.
Label top, done;
masm.bind(&top);
masm.branchSub32(Assembler::Signed, Imm32(1), scratch, &done);
{
masm.loadValue(frame.addressOfStackValue(scratch), R1);
masm.storeValue(R1, frame.addressOfStackValue(scratch, sizeof(Value)));
masm.jump(&top);
}
masm.bind(&done);
// Replace value on top of the stack with R0.
masm.storeValue(R0, frame.addressOfStackValue(-1));
return true;
}
template <>
bool BaselineCompilerCodeGen::emit_Unpick() {
frame.syncStack(0);
// Pick takes the top of the stack value and moves it under the nth value.
// For instance, unpick 2:
// before: A B C D E
// after : A B E C D
// First, move value at -1 into R0.
masm.loadValue(frame.addressOfStackValue(-1), R0);
MOZ_ASSERT(GET_INT8(handler.pc()) > 0,
"Interpreter code assumes JSOp::Unpick operand > 0");
// Move the other values up.
int32_t depth = -(GET_INT8(handler.pc()) + 1);
for (int32_t i = -1; i > depth; i--) {
Address source = frame.addressOfStackValue(i - 1);
Address dest = frame.addressOfStackValue(i);
masm.loadValue(source, R1);
masm.storeValue(R1, dest);
}
// Store R0 under the nth value.
Address dest = frame.addressOfStackValue(depth);
masm.storeValue(R0, dest);
return true;
}
template <>
bool BaselineInterpreterCodeGen::emit_Unpick() {
Register scratch = R2.scratchReg();
LoadUint8Operand(masm, scratch);
// Move the top value into R0.
masm.loadValue(frame.addressOfStackValue(-1), R0);
// Overwrite the nth stack value with R0 but first save the old value in R1.
masm.loadValue(frame.addressOfStackValue(scratch), R1);
masm.storeValue(R0, frame.addressOfStackValue(scratch));
// Now for each slot x in [n-1, 1] do the following:
//
// * Store the value in slot x in R0.
// * Store the value in the previous slot (now in R1) in slot x.
// * Move R0 to R1.
#ifdef DEBUG
// Assert the operand > 0 so the branchSub32 below doesn't "underflow" to
// negative values.
{
Label ok;
masm.branch32(Assembler::GreaterThan, scratch, Imm32(0), &ok);
masm.assumeUnreachable("JSOp::Unpick with operand <= 0?");
masm.bind(&ok);
}
#endif
Label top, done;
masm.bind(&top);
masm.branchSub32(Assembler::Zero, Imm32(1), scratch, &done);
{
// Overwrite stack slot x with slot x + 1, saving the old value in R1.
masm.loadValue(frame.addressOfStackValue(scratch), R0);
masm.storeValue(R1, frame.addressOfStackValue(scratch));
masm.moveValue(R0, R1);
masm.jump(&top);
}
// Finally, replace the value on top of the stack (slot 0) with R1. This is
// the value that used to be in slot 1.
masm.bind(&done);
masm.storeValue(R1, frame.addressOfStackValue(-1));
return true;
}
template <>
void BaselineCompilerCodeGen::emitJump() {
jsbytecode* pc = handler.pc();
MOZ_ASSERT(IsJumpOpcode(JSOp(*pc)));
frame.assertSyncedStack();
jsbytecode* target = pc + GET_JUMP_OFFSET(pc);
masm.jump(handler.labelOf(target));
}
template <>
void BaselineInterpreterCodeGen::emitJump() {
// We have to add the current pc's jump offset to the current pc. We can use
// R0 and R1 as scratch because we jump to the "next op" label so these
// registers aren't in use at this point.
Register scratch1 = R0.scratchReg();
Register scratch2 = R1.scratchReg();
Register pc = LoadBytecodePC(masm, scratch1);
LoadInt32OperandSignExtendToPtr(masm, pc, scratch2);
if (HasInterpreterPCReg()) {
masm.addPtr(scratch2, InterpreterPCReg);
} else {
masm.addPtr(pc, scratch2);
masm.storePtr(scratch2, frame.addressOfInterpreterPC());
}
masm.jump(handler.interpretOpWithPCRegLabel());
}
template <>
void BaselineCompilerCodeGen::emitTestBooleanTruthy(bool branchIfTrue,
ValueOperand val) {
jsbytecode* pc = handler.pc();
MOZ_ASSERT(IsJumpOpcode(JSOp(*pc)));
frame.assertSyncedStack();
jsbytecode* target = pc + GET_JUMP_OFFSET(pc);
masm.branchTestBooleanTruthy(branchIfTrue, val, handler.labelOf(target));
}
template <>
void BaselineInterpreterCodeGen::emitTestBooleanTruthy(bool branchIfTrue,
ValueOperand val) {
Label done;
masm.branchTestBooleanTruthy(!branchIfTrue, val, &done);
emitJump();
masm.bind(&done);
}
template <>
template <typename F1, typename F2>
[[nodiscard]] bool BaselineCompilerCodeGen::emitTestScriptFlag(
JSScript::ImmutableFlags flag, const F1& ifSet, const F2& ifNotSet,
Register scratch) {
if (handler.script()->hasFlag(flag)) {
return ifSet();
}
return ifNotSet();
}
template <>
template <typename F1, typename F2>
[[nodiscard]] bool BaselineInterpreterCodeGen::emitTestScriptFlag(
JSScript::ImmutableFlags flag, const F1& ifSet, const F2& ifNotSet,
Register scratch) {
Label flagNotSet, done;
loadScript(scratch);
masm.branchTest32(Assembler::Zero,
Address(scratch, JSScript::offsetOfImmutableFlags()),
Imm32(uint32_t(flag)), &flagNotSet);
{
if (!ifSet()) {
return false;
}
masm.jump(&done);
}
masm.bind(&flagNotSet);
{
if (!ifNotSet()) {
return false;
}
}
masm.bind(&done);
return true;
}
template <>
template <typename F>
[[nodiscard]] bool BaselineCompilerCodeGen::emitTestScriptFlag(
JSScript::ImmutableFlags flag, bool value, const F& emit,
Register scratch) {
if (handler.script()->hasFlag(flag) == value) {
return emit();
}
return true;
}
template <>
template <typename F>
[[nodiscard]] bool BaselineCompilerCodeGen::emitTestScriptFlag(
JSScript::MutableFlags flag, bool value, const F& emit, Register scratch) {
if (handler.script()->hasFlag(flag) == value) {
return emit();
}
return true;
}
template <>
template <typename F>
[[nodiscard]] bool BaselineInterpreterCodeGen::emitTestScriptFlag(
JSScript::ImmutableFlags flag, bool value, const F& emit,
Register scratch) {
Label done;
loadScript(scratch);
masm.branchTest32(value ? Assembler::Zero : Assembler::NonZero,
Address(scratch, JSScript::offsetOfImmutableFlags()),
Imm32(uint32_t(flag)), &done);
{
if (!emit()) {
return false;
}
}
masm.bind(&done);
return true;
}
template <>
template <typename F>
[[nodiscard]] bool BaselineInterpreterCodeGen::emitTestScriptFlag(
JSScript::MutableFlags flag, bool value, const F& emit, Register scratch) {
Label done;
loadScript(scratch);
masm.branchTest32(value ? Assembler::Zero : Assembler::NonZero,
Address(scratch, JSScript::offsetOfMutableFlags()),
Imm32(uint32_t(flag)), &done);
{
if (!emit()) {
return false;
}
}
masm.bind(&done);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Goto() {
frame.syncStack(0);
emitJump();
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emitTest(bool branchIfTrue) {
bool knownBoolean = frame.stackValueHasKnownType(-1, JSVAL_TYPE_BOOLEAN);
// Keep top stack value in R0.
frame.popRegsAndSync(1);
if (!knownBoolean && !emitNextIC()) {
return false;
}
// IC will leave a BooleanValue in R0, just need to branch on it.
emitTestBooleanTruthy(branchIfTrue, R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_JumpIfFalse() {
return emitTest(false);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_JumpIfTrue() {
return emitTest(true);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emitAndOr(bool branchIfTrue) {
bool knownBoolean = frame.stackValueHasKnownType(-1, JSVAL_TYPE_BOOLEAN);
// And and Or leave the original value on the stack.
frame.syncStack(0);
masm.loadValue(frame.addressOfStackValue(-1), R0);
if (!knownBoolean && !emitNextIC()) {
return false;
}
emitTestBooleanTruthy(branchIfTrue, R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_And() {
return emitAndOr(false);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Or() {
return emitAndOr(true);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Coalesce() {
// Coalesce leaves the original value on the stack.
frame.syncStack(0);
masm.loadValue(frame.addressOfStackValue(-1), R0);
Label undefinedOrNull;
masm.branchTestUndefined(Assembler::Equal, R0, &undefinedOrNull);
masm.branchTestNull(Assembler::Equal, R0, &undefinedOrNull);
emitJump();
masm.bind(&undefinedOrNull);
// fall through
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Not() {
bool knownBoolean = frame.stackValueHasKnownType(-1, JSVAL_TYPE_BOOLEAN);
// Keep top stack value in R0.
frame.popRegsAndSync(1);
if (!knownBoolean && !emitNextIC()) {
return false;
}
masm.notBoolean(R0);
frame.push(R0, JSVAL_TYPE_BOOLEAN);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Pos() {
return emitUnaryArith();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_ToNumeric() {
return emitUnaryArith();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_LoopHead() {
if (!emit_JumpTarget()) {
return false;
}
if (!emitInterruptCheck()) {
return false;
}
if (!emitWarmUpCounterIncrement()) {
return false;
}
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Void() {
frame.pop();
frame.push(UndefinedValue());
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Undefined() {
frame.push(UndefinedValue());
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Hole() {
frame.push(MagicValue(JS_ELEMENTS_HOLE));
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Null() {
frame.push(NullValue());
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_CheckIsObj() {
frame.syncStack(0);
masm.loadValue(frame.addressOfStackValue(-1), R0);
Label ok;
masm.branchTestObject(Assembler::Equal, R0, &ok);
prepareVMCall();
pushUint8BytecodeOperandArg(R0.scratchReg());
using Fn = bool (*)(JSContext*, CheckIsObjectKind);
if (!callVM<Fn, ThrowCheckIsObject>()) {
return false;
}
masm.bind(&ok);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_CheckThis() {
frame.syncStack(0);
masm.loadValue(frame.addressOfStackValue(-1), R0);
return emitCheckThis(R0);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_CheckThisReinit() {
frame.syncStack(0);
masm.loadValue(frame.addressOfStackValue(-1), R0);
return emitCheckThis(R0, /* reinit = */ true);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emitCheckThis(ValueOperand val, bool reinit) {
Label thisOK;
if (reinit) {
masm.branchTestMagic(Assembler::Equal, val, &thisOK);
} else {
masm.branchTestMagic(Assembler::NotEqual, val, &thisOK);
}
prepareVMCall();
if (reinit) {
using Fn = bool (*)(JSContext*);
if (!callVM<Fn, ThrowInitializedThis>()) {
return false;
}
} else {
using Fn = bool (*)(JSContext*);
if (!callVM<Fn, ThrowUninitializedThis>()) {
return false;
}
}
masm.bind(&thisOK);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_CheckReturn() {
MOZ_ASSERT_IF(handler.maybeScript(),
handler.maybeScript()->isDerivedClassConstructor());
// Load |this| in R0, return value in R1.
frame.popRegsAndSync(1);
emitLoadReturnValue(R1);
Label done, returnBad, checkThis;
masm.branchTestObject(Assembler::NotEqual, R1, &checkThis);
{
masm.moveValue(R1, R0);
masm.jump(&done);
}
masm.bind(&checkThis);
masm.branchTestUndefined(Assembler::NotEqual, R1, &returnBad);
masm.branchTestMagic(Assembler::NotEqual, R0, &done);
masm.bind(&returnBad);
prepareVMCall();
pushArg(R1);
using Fn = bool (*)(JSContext*, HandleValue);
if (!callVM<Fn, ThrowBadDerivedReturnOrUninitializedThis>()) {
return false;
}
masm.assumeUnreachable("Should throw on bad derived constructor return");
masm.bind(&done);
// Push |rval| or |this| onto the stack.
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_FunctionThis() {
MOZ_ASSERT_IF(handler.maybeFunction(), !handler.maybeFunction()->isArrow());
frame.pushThis();
auto boxThis = [this]() {
// Load |thisv| in R0. Skip the call if it's already an object.
Label skipCall;
frame.popRegsAndSync(1);
masm.branchTestObject(Assembler::Equal, R0, &skipCall);
prepareVMCall();
masm.loadBaselineFramePtr(FramePointer, R1.scratchReg());
pushArg(R1.scratchReg());
using Fn = bool (*)(JSContext*, BaselineFrame*, MutableHandleValue);
if (!callVM<Fn, BaselineGetFunctionThis>()) {
return false;
}
masm.bind(&skipCall);
frame.push(R0);
return true;
};
// In strict mode code, |this| is left alone.
return emitTestScriptFlag(JSScript::ImmutableFlags::Strict, false, boxThis,
R2.scratchReg());
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_GlobalThis() {
frame.syncStack(0);
loadGlobalThisValue(R0);
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_NonSyntacticGlobalThis() {
frame.syncStack(0);
prepareVMCall();
masm.loadPtr(frame.addressOfEnvironmentChain(), R0.scratchReg());
pushArg(R0.scratchReg());
using Fn = void (*)(JSContext*, HandleObject, MutableHandleValue);
if (!callVM<Fn, GetNonSyntacticGlobalThis>()) {
return false;
}
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_True() {
frame.push(BooleanValue(true));
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_False() {
frame.push(BooleanValue(false));
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Zero() {
frame.push(Int32Value(0));
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_One() {
frame.push(Int32Value(1));
return true;
}
template <>
bool BaselineCompilerCodeGen::emit_Int8() {
frame.push(Int32Value(GET_INT8(handler.pc())));
return true;
}
template <>
bool BaselineInterpreterCodeGen::emit_Int8() {
LoadInt8Operand(masm, R0.scratchReg());
masm.tagValue(JSVAL_TYPE_INT32, R0.scratchReg(), R0);
frame.push(R0);
return true;
}
template <>
bool BaselineCompilerCodeGen::emit_Int32() {
frame.push(Int32Value(GET_INT32(handler.pc())));
return true;
}
template <>
bool BaselineInterpreterCodeGen::emit_Int32() {
LoadInt32Operand(masm, R0.scratchReg());
masm.tagValue(JSVAL_TYPE_INT32, R0.scratchReg(), R0);
frame.push(R0);
return true;
}
template <>
bool BaselineCompilerCodeGen::emit_Uint16() {
frame.push(Int32Value(GET_UINT16(handler.pc())));
return true;
}
template <>
bool BaselineInterpreterCodeGen::emit_Uint16() {
LoadUint16Operand(masm, R0.scratchReg());
masm.tagValue(JSVAL_TYPE_INT32, R0.scratchReg(), R0);
frame.push(R0);
return true;
}
template <>
bool BaselineCompilerCodeGen::emit_Uint24() {
frame.push(Int32Value(GET_UINT24(handler.pc())));
return true;
}
template <>
bool BaselineInterpreterCodeGen::emit_Uint24() {
LoadUint24Operand(masm, 0, R0.scratchReg());
masm.tagValue(JSVAL_TYPE_INT32, R0.scratchReg(), R0);
frame.push(R0);
return true;
}
template <>
bool BaselineCompilerCodeGen::emit_Double() {
frame.push(GET_INLINE_VALUE(handler.pc()));
return true;
}
template <>
bool BaselineInterpreterCodeGen::emit_Double() {
LoadInlineValueOperand(masm, R0);
frame.push(R0);
return true;
}
template <>
bool BaselineCompilerCodeGen::emit_BigInt() {
BigInt* bi = handler.script()->getBigInt(handler.pc());
frame.push(BigIntValue(bi));
return true;
}
template <>
bool BaselineInterpreterCodeGen::emit_BigInt() {
Register scratch1 = R0.scratchReg();
Register scratch2 = R1.scratchReg();
loadScriptGCThing(ScriptGCThingType::BigInt, scratch1, scratch2);
masm.tagValue(JSVAL_TYPE_BIGINT, scratch1, R0);
frame.push(R0);
return true;
}
template <>
bool BaselineCompilerCodeGen::emit_String() {
frame.push(StringValue(handler.script()->getString(handler.pc())));
return true;
}
template <>
bool BaselineInterpreterCodeGen::emit_String() {
Register scratch1 = R0.scratchReg();
Register scratch2 = R1.scratchReg();
loadScriptGCThing(ScriptGCThingType::String, scratch1, scratch2);
masm.tagValue(JSVAL_TYPE_STRING, scratch1, R0);
frame.push(R0);
return true;
}
template <>
bool BaselineCompilerCodeGen::emit_Symbol() {
unsigned which = GET_UINT8(handler.pc());
JS::Symbol* sym = cx->runtime()->wellKnownSymbols->get(which);
frame.push(SymbolValue(sym));
return true;
}
template <>
bool BaselineInterpreterCodeGen::emit_Symbol() {
Register scratch1 = R0.scratchReg();
Register scratch2 = R1.scratchReg();
LoadUint8Operand(masm, scratch1);
masm.movePtr(ImmPtr(cx->runtime()->wellKnownSymbols), scratch2);
masm.loadPtr(BaseIndex(scratch2, scratch1, ScalePointer), scratch1);
masm.tagValue(JSVAL_TYPE_SYMBOL, scratch1, R0);
frame.push(R0);
return true;
}
template <>
bool BaselineCompilerCodeGen::emit_Object() {
frame.push(ObjectValue(*handler.script()->getObject(handler.pc())));
return true;
}
template <>
bool BaselineInterpreterCodeGen::emit_Object() {
Register scratch1 = R0.scratchReg();
Register scratch2 = R1.scratchReg();
loadScriptGCThing(ScriptGCThingType::Object, scratch1, scratch2);
masm.tagValue(JSVAL_TYPE_OBJECT, scratch1, R0);
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_CallSiteObj() {
return emit_Object();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_RegExp() {
prepareVMCall();
pushScriptGCThingArg(ScriptGCThingType::RegExp, R0.scratchReg(),
R1.scratchReg());
using Fn = JSObject* (*)(JSContext*, Handle<RegExpObject*>);
if (!callVM<Fn, CloneRegExpObject>()) {
return false;
}
// Box and push return value.
masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0);
frame.push(R0);
return true;
}
#ifdef ENABLE_RECORD_TUPLE
# define UNSUPPORTED_OPCODE(OP) \
template <typename Handler> \
bool BaselineCodeGen<Handler>::emit_##OP() { \
MOZ_CRASH("Record and Tuple are not supported by jit"); \
return false; \
}
UNSUPPORTED_OPCODE(InitRecord)
UNSUPPORTED_OPCODE(AddRecordProperty)
UNSUPPORTED_OPCODE(AddRecordSpread)
UNSUPPORTED_OPCODE(FinishRecord)
UNSUPPORTED_OPCODE(InitTuple)
UNSUPPORTED_OPCODE(AddTupleElement)
UNSUPPORTED_OPCODE(FinishTuple)
# undef UNSUPPORTED_OPCODE
#endif
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Lambda() {
prepareVMCall();
masm.loadPtr(frame.addressOfEnvironmentChain(), R0.scratchReg());
pushArg(R0.scratchReg());
pushScriptGCThingArg(ScriptGCThingType::Function, R0.scratchReg(),
R1.scratchReg());
using Fn = JSObject* (*)(JSContext*, HandleFunction, HandleObject);
if (!callVM<Fn, js::Lambda>()) {
return false;
}
// Box and push return value.
masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0);
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_SetFunName() {
frame.popRegsAndSync(2);
frame.push(R0);
frame.syncStack(0);
masm.unboxObject(R0, R0.scratchReg());
prepareVMCall();
pushUint8BytecodeOperandArg(R2.scratchReg());
pushArg(R1);
pushArg(R0.scratchReg());
using Fn =
bool (*)(JSContext*, HandleFunction, HandleValue, FunctionPrefixKind);
return callVM<Fn, SetFunctionName>();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_BitOr() {
return emitBinaryArith();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_BitXor() {
return emitBinaryArith();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_BitAnd() {
return emitBinaryArith();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Lsh() {
return emitBinaryArith();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Rsh() {
return emitBinaryArith();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Ursh() {
return emitBinaryArith();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Add() {
return emitBinaryArith();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Sub() {
return emitBinaryArith();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Mul() {
return emitBinaryArith();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Div() {
return emitBinaryArith();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Mod() {
return emitBinaryArith();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Pow() {
return emitBinaryArith();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emitBinaryArith() {
// Keep top JSStack value in R0 and R2
frame.popRegsAndSync(2);
// Call IC
if (!emitNextIC()) {
return false;
}
// Mark R0 as pushed stack value.
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emitUnaryArith() {
// Keep top stack value in R0.
frame.popRegsAndSync(1);
// Call IC
if (!emitNextIC()) {
return false;
}
// Mark R0 as pushed stack value.
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_BitNot() {
return emitUnaryArith();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Neg() {
return emitUnaryArith();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Inc() {
return emitUnaryArith();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Dec() {
return emitUnaryArith();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Lt() {
return emitCompare();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Le() {
return emitCompare();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Gt() {
return emitCompare();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Ge() {
return emitCompare();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Eq() {
return emitCompare();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Ne() {
return emitCompare();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emitCompare() {
// Keep top JSStack value in R0 and R1.
frame.popRegsAndSync(2);
// Call IC.
if (!emitNextIC()) {
return false;
}
// Mark R0 as pushed stack value.
frame.push(R0, JSVAL_TYPE_BOOLEAN);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_StrictEq() {
return emitCompare();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_StrictNe() {
return emitCompare();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Case() {
frame.popRegsAndSync(1);
Label done;
masm.branchTestBooleanTruthy(/* branchIfTrue */ false, R0, &done);
{
// Pop the switch value if the case matches.
masm.addToStackPtr(Imm32(sizeof(Value)));
emitJump();
}
masm.bind(&done);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Default() {
frame.pop();
return emit_Goto();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Lineno() {
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_NewArray() {
frame.syncStack(0);
if (!emitNextIC()) {
return false;
}
frame.push(R0);
return true;
}
static void MarkElementsNonPackedIfHoleValue(MacroAssembler& masm,
Register elements,
ValueOperand val) {
Label notHole;
masm.branchTestMagic(Assembler::NotEqual, val, ¬Hole);
{
Address elementsFlags(elements, ObjectElements::offsetOfFlags());
masm.or32(Imm32(ObjectElements::NON_PACKED), elementsFlags);
}
masm.bind(¬Hole);
}
template <>
bool BaselineInterpreterCodeGen::emit_InitElemArray() {
// Pop value into R0, keep the object on the stack.
frame.popRegsAndSync(1);
// Load object in R2.
Register obj = R2.scratchReg();
masm.unboxObject(frame.addressOfStackValue(-1), obj);
// Load index in R1.
Register index = R1.scratchReg();
LoadInt32Operand(masm, index);
// Store the Value. No pre-barrier because this is an initialization.
masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), obj);
masm.storeValue(R0, BaseObjectElementIndex(obj, index));
// Bump initialized length.
Address initLength(obj, ObjectElements::offsetOfInitializedLength());
masm.add32(Imm32(1), index);
masm.store32(index, initLength);
// Mark elements as NON_PACKED if we stored the hole value.
MarkElementsNonPackedIfHoleValue(masm, obj, R0);
// Post-barrier.
Label skipBarrier;
Register scratch = index;
masm.branchValueIsNurseryCell(Assembler::NotEqual, R0, scratch, &skipBarrier);
{
masm.unboxObject(frame.addressOfStackValue(-1), obj);
masm.branchPtrInNurseryChunk(Assembler::Equal, obj, scratch, &skipBarrier);
MOZ_ASSERT(obj == R2.scratchReg(), "post barrier expects object in R2");
masm.call(&postBarrierSlot_);
}
masm.bind(&skipBarrier);
return true;
}
template <>
bool BaselineCompilerCodeGen::emit_InitElemArray() {
// Pop value into R0, keep the object on the stack.
Maybe<Value> knownValue = frame.knownStackValue(-1);
frame.popRegsAndSync(1);
// Load object in R2.
Register obj = R2.scratchReg();
masm.unboxObject(frame.addressOfStackValue(-1), obj);
uint32_t index = GET_UINT32(handler.pc());
MOZ_ASSERT(index <= INT32_MAX,
"the bytecode emitter must fail to compile code that would "
"produce an index exceeding int32_t range");
// Store the Value. No pre-barrier because this is an initialization.
masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), obj);
masm.storeValue(R0, Address(obj, index * sizeof(Value)));
// Bump initialized length.
Address initLength(obj, ObjectElements::offsetOfInitializedLength());
masm.store32(Imm32(index + 1), initLength);
// Mark elements as NON_PACKED if we stored the hole value. We know this
// statically except when debugger instrumentation is enabled because that
// forces a stack-sync (which discards constants and known types) for each op.
if (knownValue && knownValue->isMagic(JS_ELEMENTS_HOLE)) {
Address elementsFlags(obj, ObjectElements::offsetOfFlags());
masm.or32(Imm32(ObjectElements::NON_PACKED), elementsFlags);
} else if (handler.compileDebugInstrumentation()) {
MarkElementsNonPackedIfHoleValue(masm, obj, R0);
} else {
#ifdef DEBUG
Label notHole;
masm.branchTestMagic(Assembler::NotEqual, R0, ¬Hole);
masm.assumeUnreachable("Unexpected hole value");
masm.bind(¬Hole);
#endif
}
// Post-barrier.
if (knownValue) {
MOZ_ASSERT(JS::GCPolicy<Value>::isTenured(*knownValue));
} else {
Label skipBarrier;
Register scratch = R1.scratchReg();
masm.branchValueIsNurseryCell(Assembler::NotEqual, R0, scratch,
&skipBarrier);
{
masm.unboxObject(frame.addressOfStackValue(-1), obj);
masm.branchPtrInNurseryChunk(Assembler::Equal, obj, scratch,
&skipBarrier);
MOZ_ASSERT(obj == R2.scratchReg(), "post barrier expects object in R2");
masm.call(&postBarrierSlot_);
}
masm.bind(&skipBarrier);
}
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_NewObject() {
return emitNewObject();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_NewInit() {
return emitNewObject();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emitNewObject() {
frame.syncStack(0);
if (!emitNextIC()) {
return false;
}
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_InitElem() {
// Store RHS in the scratch slot.
frame.storeStackValue(-1, frame.addressOfScratchValue(), R2);
frame.pop();
// Keep object and index in R0 and R1.
frame.popRegsAndSync(2);
// Push the object to store the result of the IC.
frame.push(R0);
frame.syncStack(0);
// Keep RHS on the stack.
frame.pushScratchValue();
// Call IC.
if (!emitNextIC()) {
return false;
}
// Pop the rhs, so that the object is on the top of the stack.
frame.pop();
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_InitHiddenElem() {
return emit_InitElem();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_InitLockedElem() {
return emit_InitElem();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_MutateProto() {
// Keep values on the stack for the decompiler.
frame.syncStack(0);
masm.unboxObject(frame.addressOfStackValue(-2), R0.scratchReg());
masm.loadValue(frame.addressOfStackValue(-1), R1);
prepareVMCall();
pushArg(R1);
pushArg(R0.scratchReg());
using Fn = bool (*)(JSContext*, Handle<PlainObject*>, HandleValue);
if (!callVM<Fn, MutatePrototype>()) {
return false;
}
frame.pop();
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_InitProp() {
// Load lhs in R0, rhs in R1.
frame.syncStack(0);
masm.loadValue(frame.addressOfStackValue(-2), R0);
masm.loadValue(frame.addressOfStackValue(-1), R1);
// Call IC.
if (!emitNextIC()) {
return false;
}
// Leave the object on the stack.
frame.pop();
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_InitLockedProp() {
return emit_InitProp();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_InitHiddenProp() {
return emit_InitProp();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_GetElem() {
// Keep top two stack values in R0 and R1.
frame.popRegsAndSync(2);
// Call IC.
if (!emitNextIC()) {
return false;
}
// Mark R0 as pushed stack value.
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_GetElemSuper() {
// Store obj in the scratch slot.
frame.storeStackValue(-1, frame.addressOfScratchValue(), R2);
frame.pop();
// Keep receiver and index in R0 and R1.
frame.popRegsAndSync(2);
// Keep obj on the stack.
frame.pushScratchValue();
if (!emitNextIC()) {
return false;
}
frame.pop();
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_SetElem() {
// Store RHS in the scratch slot.
frame.storeStackValue(-1, frame.addressOfScratchValue(), R2);
frame.pop();
// Keep object and index in R0 and R1.
frame.popRegsAndSync(2);
// Keep RHS on the stack.
frame.pushScratchValue();
// Call IC.
if (!emitNextIC()) {
return false;
}
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_StrictSetElem() {
return emit_SetElem();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emitSetElemSuper(bool strict) {
// Incoming stack is |receiver, propval, obj, rval|. We need to shuffle
// stack to leave rval when operation is complete.
// Pop rval into R0, then load receiver into R1 and replace with rval.
frame.popRegsAndSync(1);
masm.loadValue(frame.addressOfStackValue(-3), R1);
masm.storeValue(R0, frame.addressOfStackValue(-3));
prepareVMCall();
pushArg(Imm32(strict));
pushArg(R0); // rval
masm.loadValue(frame.addressOfStackValue(-2), R0);
pushArg(R0); // propval
pushArg(R1); // receiver
masm.loadValue(frame.addressOfStackValue(-1), R0);
pushArg(R0); // obj
using Fn = bool (*)(JSContext*, HandleValue, HandleValue, HandleValue,
HandleValue, bool);
if (!callVM<Fn, js::SetElementSuper>()) {
return false;
}
frame.popn(2);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_SetElemSuper() {
return emitSetElemSuper(/* strict = */ false);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_StrictSetElemSuper() {
return emitSetElemSuper(/* strict = */ true);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emitDelElem(bool strict) {
// Keep values on the stack for the decompiler.
frame.syncStack(0);
masm.loadValue(frame.addressOfStackValue(-2), R0);
masm.loadValue(frame.addressOfStackValue(-1), R1);
prepareVMCall();
pushArg(R1);
pushArg(R0);
using Fn = bool (*)(JSContext*, HandleValue, HandleValue, bool*);
if (strict) {
if (!callVM<Fn, DelElemOperation<true>>()) {
return false;
}
} else {
if (!callVM<Fn, DelElemOperation<false>>()) {
return false;
}
}
masm.boxNonDouble(JSVAL_TYPE_BOOLEAN, ReturnReg, R1);
frame.popn(2);
frame.push(R1, JSVAL_TYPE_BOOLEAN);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_DelElem() {
return emitDelElem(/* strict = */ false);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_StrictDelElem() {
return emitDelElem(/* strict = */ true);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_In() {
frame.popRegsAndSync(2);
if (!emitNextIC()) {
return false;
}
frame.push(R0, JSVAL_TYPE_BOOLEAN);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_HasOwn() {
frame.popRegsAndSync(2);
if (!emitNextIC()) {
return false;
}
frame.push(R0, JSVAL_TYPE_BOOLEAN);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_CheckPrivateField() {
// Keep key and val on the stack.
frame.syncStack(0);
masm.loadValue(frame.addressOfStackValue(-2), R0);
masm.loadValue(frame.addressOfStackValue(-1), R1);
if (!emitNextIC()) {
return false;
}
frame.push(R0, JSVAL_TYPE_BOOLEAN);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_NewPrivateName() {
prepareVMCall();
pushScriptNameArg(R0.scratchReg(), R1.scratchReg());
using Fn = JS::Symbol* (*)(JSContext*, Handle<JSAtom*>);
if (!callVM<Fn, NewPrivateName>()) {
return false;
}
masm.tagValue(JSVAL_TYPE_SYMBOL, ReturnReg, R0);
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_GetGName() {
frame.syncStack(0);
loadGlobalLexicalEnvironment(R0.scratchReg());
// Call IC.
if (!emitNextIC()) {
return false;
}
// Mark R0 as pushed stack value.
frame.push(R0);
return true;
}
template <>
bool BaselineCompilerCodeGen::tryOptimizeBindGlobalName() {
JSScript* script = handler.script();
MOZ_ASSERT(!script->hasNonSyntacticScope());
Rooted<GlobalObject*> global(cx, &script->global());
Rooted<PropertyName*> name(cx, script->getName(handler.pc()));
if (JSObject* binding = MaybeOptimizeBindGlobalName(cx, global, name)) {
frame.push(ObjectValue(*binding));
return true;
}
return false;
}
template <>
bool BaselineInterpreterCodeGen::tryOptimizeBindGlobalName() {
// Interpreter doesn't optimize simple BindGNames.
return false;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_BindGName() {
if (tryOptimizeBindGlobalName()) {
return true;
}
frame.syncStack(0);
loadGlobalLexicalEnvironment(R0.scratchReg());
// Call IC.
if (!emitNextIC()) {
return false;
}
// Mark R0 as pushed stack value.
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_BindVar() {
frame.syncStack(0);
masm.loadPtr(frame.addressOfEnvironmentChain(), R0.scratchReg());
prepareVMCall();
pushArg(R0.scratchReg());
using Fn = JSObject* (*)(JSContext*, JSObject*);
if (!callVM<Fn, BindVarOperation>()) {
return false;
}
masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0);
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_SetProp() {
// Keep lhs in R0, rhs in R1.
frame.popRegsAndSync(2);
// Keep RHS on the stack.
frame.push(R1);
frame.syncStack(0);
// Call IC.
if (!emitNextIC()) {
return false;
}
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_StrictSetProp() {
return emit_SetProp();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_SetName() {
return emit_SetProp();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_StrictSetName() {
return emit_SetProp();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_SetGName() {
return emit_SetProp();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_StrictSetGName() {
return emit_SetProp();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emitSetPropSuper(bool strict) {
// Incoming stack is |receiver, obj, rval|. We need to shuffle stack to
// leave rval when operation is complete.
// Pop rval into R0, then load receiver into R1 and replace with rval.
frame.popRegsAndSync(1);
masm.loadValue(frame.addressOfStackValue(-2), R1);
masm.storeValue(R0, frame.addressOfStackValue(-2));
prepareVMCall();
pushArg(Imm32(strict));
pushArg(R0); // rval
pushScriptNameArg(R0.scratchReg(), R2.scratchReg());
pushArg(R1); // receiver
masm.loadValue(frame.addressOfStackValue(-1), R0);
pushArg(R0); // obj
using Fn = bool (*)(JSContext*, HandleValue, HandleValue,
Handle<PropertyName*>, HandleValue, bool);
if (!callVM<Fn, js::SetPropertySuper>()) {
return false;
}
frame.pop();
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_SetPropSuper() {
return emitSetPropSuper(/* strict = */ false);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_StrictSetPropSuper() {
return emitSetPropSuper(/* strict = */ true);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_GetProp() {
// Keep object in R0.
frame.popRegsAndSync(1);
// Call IC.
if (!emitNextIC()) {
return false;
}
// Mark R0 as pushed stack value.
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_GetBoundName() {
return emit_GetProp();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_GetPropSuper() {
// Receiver -> R1, ObjectOrNull -> R0
frame.popRegsAndSync(1);
masm.loadValue(frame.addressOfStackValue(-1), R1);
frame.pop();
if (!emitNextIC()) {
return false;
}
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emitDelProp(bool strict) {
// Keep value on the stack for the decompiler.
frame.syncStack(0);
masm.loadValue(frame.addressOfStackValue(-1), R0);
prepareVMCall();
pushScriptNameArg(R1.scratchReg(), R2.scratchReg());
pushArg(R0);
using Fn = bool (*)(JSContext*, HandleValue, Handle<PropertyName*>, bool*);
if (strict) {
if (!callVM<Fn, DelPropOperation<true>>()) {
return false;
}
} else {
if (!callVM<Fn, DelPropOperation<false>>()) {
return false;
}
}
masm.boxNonDouble(JSVAL_TYPE_BOOLEAN, ReturnReg, R1);
frame.pop();
frame.push(R1, JSVAL_TYPE_BOOLEAN);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_DelProp() {
return emitDelProp(/* strict = */ false);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_StrictDelProp() {
return emitDelProp(/* strict = */ true);
}
template <>
void BaselineCompilerCodeGen::getEnvironmentCoordinateObject(Register reg) {
EnvironmentCoordinate ec(handler.pc());
masm.loadPtr(frame.addressOfEnvironmentChain(), reg);
for (unsigned i = ec.hops(); i; i--) {
masm.unboxObject(
Address(reg, EnvironmentObject::offsetOfEnclosingEnvironment()), reg);
}
}
template <>
void BaselineInterpreterCodeGen::getEnvironmentCoordinateObject(Register reg) {
MOZ_CRASH("Shouldn't call this for interpreter");
}
template <>
Address BaselineCompilerCodeGen::getEnvironmentCoordinateAddressFromObject(
Register objReg, Register reg) {
EnvironmentCoordinate ec(handler.pc());
if (EnvironmentObject::nonExtensibleIsFixedSlot(ec)) {
return Address(objReg, NativeObject::getFixedSlotOffset(ec.slot()));
}
uint32_t slot = EnvironmentObject::nonExtensibleDynamicSlotIndex(ec);
masm.loadPtr(Address(objReg, NativeObject::offsetOfSlots()), reg);
return Address(reg, slot * sizeof(Value));
}
template <>
Address BaselineInterpreterCodeGen::getEnvironmentCoordinateAddressFromObject(
Register objReg, Register reg) {
MOZ_CRASH("Shouldn't call this for interpreter");
}
template <typename Handler>
Address BaselineCodeGen<Handler>::getEnvironmentCoordinateAddress(
Register reg) {
getEnvironmentCoordinateObject(reg);
return getEnvironmentCoordinateAddressFromObject(reg, reg);
}
// For a JOF_ENVCOORD op load the number of hops from the bytecode and skip this
// number of environment objects.
static void LoadAliasedVarEnv(MacroAssembler& masm, Register env,
Register scratch) {
static_assert(ENVCOORD_HOPS_LEN == 1,
"Code assumes number of hops is stored in uint8 operand");
LoadUint8Operand(masm, scratch);
Label top, done;
masm.branchTest32(Assembler::Zero, scratch, scratch, &done);
masm.bind(&top);
{
Address nextEnv(env, EnvironmentObject::offsetOfEnclosingEnvironment());
masm.unboxObject(nextEnv, env);
masm.branchSub32(Assembler::NonZero, Imm32(1), scratch, &top);
}
masm.bind(&done);
}
template <>
void BaselineCompilerCodeGen::emitGetAliasedVar(ValueOperand dest) {
frame.syncStack(0);
Address address = getEnvironmentCoordinateAddress(R0.scratchReg());
masm.loadValue(address, dest);
}
template <>
void BaselineInterpreterCodeGen::emitGetAliasedVar(ValueOperand dest) {
Register env = R0.scratchReg();
Register scratch = R1.scratchReg();
// Load the right environment object.
masm.loadPtr(frame.addressOfEnvironmentChain(), env);
LoadAliasedVarEnv(masm, env, scratch);
// Load the slot index.
static_assert(ENVCOORD_SLOT_LEN == 3,
"Code assumes slot is stored in uint24 operand");
LoadUint24Operand(masm, ENVCOORD_HOPS_LEN, scratch);
// Load the Value from a fixed or dynamic slot.
// See EnvironmentObject::nonExtensibleIsFixedSlot.
Label isDynamic, done;
masm.branch32(Assembler::AboveOrEqual, scratch,
Imm32(NativeObject::MAX_FIXED_SLOTS), &isDynamic);
{
uint32_t offset = NativeObject::getFixedSlotOffset(0);
masm.loadValue(BaseValueIndex(env, scratch, offset), dest);
masm.jump(&done);
}
masm.bind(&isDynamic);
{
masm.loadPtr(Address(env, NativeObject::offsetOfSlots()), env);
// Use an offset to subtract the number of fixed slots.
int32_t offset = -int32_t(NativeObject::MAX_FIXED_SLOTS * sizeof(Value));
masm.loadValue(BaseValueIndex(env, scratch, offset), dest);
}
masm.bind(&done);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emitGetAliasedDebugVar(ValueOperand dest) {
frame.syncStack(0);
Register env = R0.scratchReg();
// Load the right environment object.
masm.loadPtr(frame.addressOfEnvironmentChain(), env);
prepareVMCall();
pushBytecodePCArg();
pushArg(env);
using Fn =
bool (*)(JSContext*, JSObject* env, jsbytecode*, MutableHandleValue);
return callVM<Fn, LoadAliasedDebugVar>();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_GetAliasedDebugVar() {
if (!emitGetAliasedDebugVar(R0)) {
return false;
}
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_GetAliasedVar() {
emitGetAliasedVar(R0);
frame.push(R0);
return true;
}
template <>
bool BaselineCompilerCodeGen::emit_SetAliasedVar() {
// Keep rvalue in R0.
frame.popRegsAndSync(1);
Register objReg = R2.scratchReg();
getEnvironmentCoordinateObject(objReg);
Address address =
getEnvironmentCoordinateAddressFromObject(objReg, R1.scratchReg());
masm.guardedCallPreBarrier(address, MIRType::Value);
masm.storeValue(R0, address);
frame.push(R0);
// Only R0 is live at this point.
// Scope coordinate object is already in R2.scratchReg().
Register temp = R1.scratchReg();
Label skipBarrier;
masm.branchPtrInNurseryChunk(Assembler::Equal, objReg, temp, &skipBarrier);
masm.branchValueIsNurseryCell(Assembler::NotEqual, R0, temp, &skipBarrier);
masm.call(&postBarrierSlot_); // Won't clobber R0
masm.bind(&skipBarrier);
return true;
}
template <>
bool BaselineInterpreterCodeGen::emit_SetAliasedVar() {
AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
MOZ_ASSERT(!regs.has(FramePointer));
regs.take(R2);
if (HasInterpreterPCReg()) {
regs.take(InterpreterPCReg);
}
Register env = regs.takeAny();
Register scratch1 = regs.takeAny();
Register scratch2 = regs.takeAny();
Register scratch3 = regs.takeAny();
// Load the right environment object.
masm.loadPtr(frame.addressOfEnvironmentChain(), env);
LoadAliasedVarEnv(masm, env, scratch1);
// Load the slot index.
static_assert(ENVCOORD_SLOT_LEN == 3,
"Code assumes slot is stored in uint24 operand");
LoadUint24Operand(masm, ENVCOORD_HOPS_LEN, scratch1);
// Store the RHS Value in R2.
masm.loadValue(frame.addressOfStackValue(-1), R2);
// Load a pointer to the fixed or dynamic slot into scratch2. We want to call
// guardedCallPreBarrierAnyZone once to avoid code bloat.
// See EnvironmentObject::nonExtensibleIsFixedSlot.
Label isDynamic, done;
masm.branch32(Assembler::AboveOrEqual, scratch1,
Imm32(NativeObject::MAX_FIXED_SLOTS), &isDynamic);
{
uint32_t offset = NativeObject::getFixedSlotOffset(0);
BaseValueIndex slotAddr(env, scratch1, offset);
masm.computeEffectiveAddress(slotAddr, scratch2);
masm.jump(&done);
}
masm.bind(&isDynamic);
{
masm.loadPtr(Address(env, NativeObject::offsetOfSlots()), scratch2);
// Use an offset to subtract the number of fixed slots.
int32_t offset = -int32_t(NativeObject::MAX_FIXED_SLOTS * sizeof(Value));
BaseValueIndex slotAddr(scratch2, scratch1, offset);
masm.computeEffectiveAddress(slotAddr, scratch2);
}
masm.bind(&done);
// Pre-barrier and store.
Address slotAddr(scratch2, 0);
masm.guardedCallPreBarrierAnyZone(slotAddr, MIRType::Value, scratch3);
masm.storeValue(R2, slotAddr);
// Post barrier.
Label skipBarrier;
masm.branchPtrInNurseryChunk(Assembler::Equal, env, scratch1, &skipBarrier);
masm.branchValueIsNurseryCell(Assembler::NotEqual, R2, scratch1,
&skipBarrier);
{
// Post barrier code expects the object in R2.
masm.movePtr(env, R2.scratchReg());
masm.call(&postBarrierSlot_);
}
masm.bind(&skipBarrier);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_GetName() {
frame.syncStack(0);
masm.loadPtr(frame.addressOfEnvironmentChain(), R0.scratchReg());
// Call IC.
if (!emitNextIC()) {
return false;
}
// Mark R0 as pushed stack value.
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_BindName() {
frame.syncStack(0);
masm.loadPtr(frame.addressOfEnvironmentChain(), R0.scratchReg());
// Call IC.
if (!emitNextIC()) {
return false;
}
// Mark R0 as pushed stack value.
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_DelName() {
frame.syncStack(0);
masm.loadPtr(frame.addressOfEnvironmentChain(), R0.scratchReg());
prepareVMCall();
pushArg(R0.scratchReg());
pushScriptNameArg(R1.scratchReg(), R2.scratchReg());
using Fn = bool (*)(JSContext*, Handle<PropertyName*>, HandleObject,
MutableHandleValue);
if (!callVM<Fn, js::DeleteNameOperation>()) {
return false;
}
frame.push(R0);
return true;
}
template <>
bool BaselineCompilerCodeGen::emit_GetImport() {
JSScript* script = handler.script();
ModuleEnvironmentObject* env = GetModuleEnvironmentForScript(script);
MOZ_ASSERT(env);
jsid id = NameToId(script->getName(handler.pc()));
ModuleEnvironmentObject* targetEnv;
Maybe<PropertyInfo> prop;
MOZ_ALWAYS_TRUE(env->lookupImport(id, &targetEnv, &prop));
frame.syncStack(0);
uint32_t slot = prop->slot();
Register scratch = R0.scratchReg();
masm.movePtr(ImmGCPtr(targetEnv), scratch);
if (slot < targetEnv->numFixedSlots()) {
masm.loadValue(Address(scratch, NativeObject::getFixedSlotOffset(slot)),
R0);
} else {
masm.loadPtr(Address(scratch, NativeObject::offsetOfSlots()), scratch);
masm.loadValue(
Address(scratch, (slot - targetEnv->numFixedSlots()) * sizeof(Value)),
R0);
}
// Imports are initialized by this point except in rare circumstances, so
// don't emit a check unless we have to.
if (targetEnv->getSlot(slot).isMagic(JS_UNINITIALIZED_LEXICAL)) {
if (!emitUninitializedLexicalCheck(R0)) {
return false;
}
}
frame.push(R0);
return true;
}
template <>
bool BaselineInterpreterCodeGen::emit_GetImport() {
frame.syncStack(0);
masm.loadPtr(frame.addressOfEnvironmentChain(), R0.scratchReg());
prepareVMCall();
pushBytecodePCArg();
pushScriptArg();
pushArg(R0.scratchReg());
using Fn = bool (*)(JSContext*, HandleObject, HandleScript, jsbytecode*,
MutableHandleValue);
if (!callVM<Fn, GetImportOperation>()) {
return false;
}
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_GetIntrinsic() {
frame.syncStack(0);
if (!emitNextIC()) {
return false;
}
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_SetIntrinsic() {
frame.syncStack(0);
masm.loadValue(frame.addressOfStackValue(-1), R0);
prepareVMCall();
pushArg(R0);
pushBytecodePCArg();
pushScriptArg();
using Fn = bool (*)(JSContext*, JSScript*, jsbytecode*, HandleValue);
return callVM<Fn, SetIntrinsicOperation>();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_GlobalOrEvalDeclInstantiation() {
frame.syncStack(0);
prepareVMCall();
loadInt32LengthBytecodeOperand(R0.scratchReg());
pushArg(R0.scratchReg());
pushScriptArg();
masm.loadPtr(frame.addressOfEnvironmentChain(), R0.scratchReg());
pushArg(R0.scratchReg());
using Fn = bool (*)(JSContext*, HandleObject, HandleScript, GCThingIndex);
return callVM<Fn, js::GlobalOrEvalDeclInstantiation>();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emitInitPropGetterSetter() {
// Keep values on the stack for the decompiler.
frame.syncStack(0);
prepareVMCall();
masm.unboxObject(frame.addressOfStackValue(-1), R0.scratchReg());
masm.unboxObject(frame.addressOfStackValue(-2), R1.scratchReg());
pushArg(R0.scratchReg());
pushScriptNameArg(R0.scratchReg(), R2.scratchReg());
pushArg(R1.scratchReg());
pushBytecodePCArg();
using Fn = bool (*)(JSContext*, jsbytecode*, HandleObject,
Handle<PropertyName*>, HandleObject);
if (!callVM<Fn, InitPropGetterSetterOperation>()) {
return false;
}
frame.pop();
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_InitPropGetter() {
return emitInitPropGetterSetter();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_InitHiddenPropGetter() {
return emitInitPropGetterSetter();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_InitPropSetter() {
return emitInitPropGetterSetter();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_InitHiddenPropSetter() {
return emitInitPropGetterSetter();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emitInitElemGetterSetter() {
// Load index and value in R0 and R1, but keep values on the stack for the
// decompiler.
frame.syncStack(0);
masm.loadValue(frame.addressOfStackValue(-2), R0);
masm.unboxObject(frame.addressOfStackValue(-1), R1.scratchReg());
prepareVMCall();
pushArg(R1.scratchReg());
pushArg(R0);
masm.unboxObject(frame.addressOfStackValue(-3), R0.scratchReg());
pushArg(R0.scratchReg());
pushBytecodePCArg();
using Fn = bool (*)(JSContext*, jsbytecode*, HandleObject, HandleValue,
HandleObject);
if (!callVM<Fn, InitElemGetterSetterOperation>()) {
return false;
}
frame.popn(2);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_InitElemGetter() {
return emitInitElemGetterSetter();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_InitHiddenElemGetter() {
return emitInitElemGetterSetter();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_InitElemSetter() {
return emitInitElemGetterSetter();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_InitHiddenElemSetter() {
return emitInitElemGetterSetter();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_InitElemInc() {
// Keep the object and rhs on the stack.
frame.syncStack(0);
// Load object in R0, index in R1.
masm.loadValue(frame.addressOfStackValue(-3), R0);
masm.loadValue(frame.addressOfStackValue(-2), R1);
// Call IC.
if (!emitNextIC()) {
return false;
}
// Pop the rhs
frame.pop();
// Increment index
Address indexAddr = frame.addressOfStackValue(-1);
#ifdef DEBUG
Label isInt32;
masm.branchTestInt32(Assembler::Equal, indexAddr, &isInt32);
masm.assumeUnreachable("INITELEM_INC index must be Int32");
masm.bind(&isInt32);
#endif
masm.incrementInt32Value(indexAddr);
return true;
}
template <>
bool BaselineCompilerCodeGen::emit_GetLocal() {
frame.pushLocal(GET_LOCALNO(handler.pc()));
return true;
}
static BaseValueIndex ComputeAddressOfLocal(MacroAssembler& masm,
Register indexScratch) {
// Locals are stored in memory at a negative offset from the frame pointer. We
// negate the index first to effectively subtract it.
masm.negPtr(indexScratch);
return BaseValueIndex(FramePointer, indexScratch,
BaselineFrame::reverseOffsetOfLocal(0));
}
template <>
bool BaselineInterpreterCodeGen::emit_GetLocal() {
Register scratch = R0.scratchReg();
LoadUint24Operand(masm, 0, scratch);
BaseValueIndex addr = ComputeAddressOfLocal(masm, scratch);
masm.loadValue(addr, R0);
frame.push(R0);
return true;
}
template <>
bool BaselineCompilerCodeGen::emit_SetLocal() {
// Ensure no other StackValue refers to the old value, for instance i + (i =
// 3). This also allows us to use R0 as scratch below.
frame.syncStack(1);
uint32_t local = GET_LOCALNO(handler.pc());
frame.storeStackValue(-1, frame.addressOfLocal(local), R0);
return true;
}
template <>
bool BaselineInterpreterCodeGen::emit_SetLocal() {
Register scratch = R0.scratchReg();
LoadUint24Operand(masm, 0, scratch);
BaseValueIndex addr = ComputeAddressOfLocal(masm, scratch);
masm.loadValue(frame.addressOfStackValue(-1), R1);
masm.storeValue(R1, addr);
return true;
}
template <>
bool BaselineCompilerCodeGen::emitFormalArgAccess(JSOp op) {
MOZ_ASSERT(op == JSOp::GetArg || op == JSOp::SetArg);
uint32_t arg = GET_ARGNO(handler.pc());
// Fast path: the script does not use |arguments| or formals don't
// alias the arguments object.
if (!handler.script()->argsObjAliasesFormals()) {
if (op == JSOp::GetArg) {
frame.pushArg(arg);
} else {
// See the comment in emit_SetLocal.
frame.syncStack(1);
frame.storeStackValue(-1, frame.addressOfArg(arg), R0);
}
return true;
}
// Sync so that we can use R0.
frame.syncStack(0);
// Load the arguments object data vector.
Register reg = R2.scratchReg();
masm.loadPtr(frame.addressOfArgsObj(), reg);
masm.loadPrivate(Address(reg, ArgumentsObject::getDataSlotOffset()), reg);
// Load/store the argument.
Address argAddr(reg, ArgumentsData::offsetOfArgs() + arg * sizeof(Value));
if (op == JSOp::GetArg) {
masm.loadValue(argAddr, R0);
frame.push(R0);
} else {
Register temp = R1.scratchReg();
masm.guardedCallPreBarrierAnyZone(argAddr, MIRType::Value, temp);
masm.loadValue(frame.addressOfStackValue(-1), R0);
masm.storeValue(R0, argAddr);
MOZ_ASSERT(frame.numUnsyncedSlots() == 0);
// Reload the arguments object.
Register reg = R2.scratchReg();
masm.loadPtr(frame.addressOfArgsObj(), reg);
Label skipBarrier;
masm.branchPtrInNurseryChunk(Assembler::Equal, reg, temp, &skipBarrier);
masm.branchValueIsNurseryCell(Assembler::NotEqual, R0, temp, &skipBarrier);
masm.call(&postBarrierSlot_);
masm.bind(&skipBarrier);
}
return true;
}
template <>
bool BaselineInterpreterCodeGen::emitFormalArgAccess(JSOp op) {
MOZ_ASSERT(op == JSOp::GetArg || op == JSOp::SetArg);
// Load the index.
Register argReg = R1.scratchReg();
LoadUint16Operand(masm, argReg);
// If the frame has no arguments object, this must be an unaliased access.
Label isUnaliased, done;
masm.branchTest32(Assembler::Zero, frame.addressOfFlags(),
Imm32(BaselineFrame::HAS_ARGS_OBJ), &isUnaliased);
{
Register reg = R2.scratchReg();
// If it's an unmapped arguments object, this is an unaliased access.
loadScript(reg);
masm.branchTest32(
Assembler::Zero, Address(reg, JSScript::offsetOfImmutableFlags()),
Imm32(uint32_t(JSScript::ImmutableFlags::HasMappedArgsObj)),
&isUnaliased);
// Load the arguments object data vector.
masm.loadPtr(frame.addressOfArgsObj(), reg);
masm.loadPrivate(Address(reg, ArgumentsObject::getDataSlotOffset()), reg);
// Load/store the argument.
BaseValueIndex argAddr(reg, argReg, ArgumentsData::offsetOfArgs());
if (op == JSOp::GetArg) {
masm.loadValue(argAddr, R0);
frame.push(R0);
} else {
masm.guardedCallPreBarrierAnyZone(argAddr, MIRType::Value,
R0.scratchReg());
masm.loadValue(frame.addressOfStackValue(-1), R0);
masm.storeValue(R0, argAddr);
// Reload the arguments object.
masm.loadPtr(frame.addressOfArgsObj(), reg);
Register temp = R1.scratchReg();
masm.branchPtrInNurseryChunk(Assembler::Equal, reg, temp, &done);
masm.branchValueIsNurseryCell(Assembler::NotEqual, R0, temp, &done);
masm.call(&postBarrierSlot_);
}
masm.jump(&done);
}
masm.bind(&isUnaliased);
{
BaseValueIndex addr(FramePointer, argReg,
JitFrameLayout::offsetOfActualArgs());
if (op == JSOp::GetArg) {
masm.loadValue(addr, R0);
frame.push(R0);
} else {
masm.loadValue(frame.addressOfStackValue(-1), R0);
masm.storeValue(R0, addr);
}
}
masm.bind(&done);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_GetArg() {
return emitFormalArgAccess(JSOp::GetArg);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_SetArg() {
return emitFormalArgAccess(JSOp::SetArg);
}
template <>
bool BaselineInterpreterCodeGen::emit_GetFrameArg() {
frame.syncStack(0);
Register argReg = R1.scratchReg();
LoadUint16Operand(masm, argReg);
BaseValueIndex addr(FramePointer, argReg,
JitFrameLayout::offsetOfActualArgs());
masm.loadValue(addr, R0);
frame.push(R0);
return true;
}
template <>
bool BaselineCompilerCodeGen::emit_GetFrameArg() {
uint32_t arg = GET_ARGNO(handler.pc());
frame.pushArg(arg);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_ArgumentsLength() {
frame.syncStack(0);
masm.loadNumActualArgs(FramePointer, R0.scratchReg());
masm.tagValue(JSVAL_TYPE_INT32, R0.scratchReg(), R0);
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_GetActualArg() {
frame.popRegsAndSync(1);
#ifdef DEBUG
{
Label ok;
masm.branchTestInt32(Assembler::Equal, R0, &ok);
masm.assumeUnreachable("GetActualArg unexpected type");
masm.bind(&ok);
}
#endif
Register index = R0.scratchReg();
masm.unboxInt32(R0, index);
#ifdef DEBUG
{
Label ok;
masm.loadNumActualArgs(FramePointer, R1.scratchReg());
masm.branch32(Assembler::Above, R1.scratchReg(), index, &ok);
masm.assumeUnreachable("GetActualArg invalid index");
masm.bind(&ok);
}
#endif
BaseValueIndex addr(FramePointer, index,
JitFrameLayout::offsetOfActualArgs());
masm.loadValue(addr, R0);
frame.push(R0);
return true;
}
template <>
void BaselineCompilerCodeGen::loadNumFormalArguments(Register dest) {
masm.move32(Imm32(handler.function()->nargs()), dest);
}
template <>
void BaselineInterpreterCodeGen::loadNumFormalArguments(Register dest) {
masm.loadFunctionFromCalleeToken(frame.addressOfCalleeToken(), dest);
masm.loadFunctionArgCount(dest, dest);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_NewTarget() {
MOZ_ASSERT_IF(handler.maybeFunction(), !handler.maybeFunction()->isArrow());
frame.syncStack(0);
#ifdef DEBUG
Register scratch1 = R0.scratchReg();
Register scratch2 = R1.scratchReg();
Label isFunction;
masm.loadPtr(frame.addressOfCalleeToken(), scratch1);
masm.branchTestPtr(Assembler::Zero, scratch1, Imm32(CalleeTokenScriptBit),
&isFunction);
masm.assumeUnreachable("Unexpected non-function script");
masm.bind(&isFunction);
Label notArrow;
masm.andPtr(Imm32(uint32_t(CalleeTokenMask)), scratch1);
masm.branchFunctionKind(Assembler::NotEqual,
FunctionFlags::FunctionKind::Arrow, scratch1,
scratch2, ¬Arrow);
masm.assumeUnreachable("Unexpected arrow function");
masm.bind(¬Arrow);
#endif
// if (isConstructing()) push(argv[Max(numActualArgs, numFormalArgs)])
Label notConstructing, done;
masm.branchTestPtr(Assembler::Zero, frame.addressOfCalleeToken(),
Imm32(CalleeToken_FunctionConstructing), ¬Constructing);
{
Register argvLen = R0.scratchReg();
Register nformals = R1.scratchReg();
masm.loadNumActualArgs(FramePointer, argvLen);
// If argvLen < nformals, set argvlen := nformals.
loadNumFormalArguments(nformals);
masm.cmp32Move32(Assembler::Below, argvLen, nformals, nformals, argvLen);
BaseValueIndex newTarget(FramePointer, argvLen,
JitFrameLayout::offsetOfActualArgs());
masm.loadValue(newTarget, R0);
masm.jump(&done);
}
// else push(undefined)
masm.bind(¬Constructing);
masm.moveValue(UndefinedValue(), R0);
masm.bind(&done);
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_ThrowSetConst() {
prepareVMCall();
pushArg(Imm32(JSMSG_BAD_CONST_ASSIGN));
using Fn = bool (*)(JSContext*, unsigned);
return callVM<Fn, jit::ThrowRuntimeLexicalError>();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emitUninitializedLexicalCheck(
const ValueOperand& val) {
Label done;
masm.branchTestMagicValue(Assembler::NotEqual, val, JS_UNINITIALIZED_LEXICAL,
&done);
prepareVMCall();
pushArg(Imm32(JSMSG_UNINITIALIZED_LEXICAL));
using Fn = bool (*)(JSContext*, unsigned);
if (!callVM<Fn, jit::ThrowRuntimeLexicalError>()) {
return false;
}
masm.bind(&done);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_CheckLexical() {
frame.syncStack(0);
masm.loadValue(frame.addressOfStackValue(-1), R0);
return emitUninitializedLexicalCheck(R0);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_CheckAliasedLexical() {
return emit_CheckLexical();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_InitLexical() {
return emit_SetLocal();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_InitGLexical() {
frame.popRegsAndSync(1);
pushGlobalLexicalEnvironmentValue(R1);
frame.push(R0);
return emit_SetProp();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_InitAliasedLexical() {
return emit_SetAliasedVar();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Uninitialized() {
frame.push(MagicValue(JS_UNINITIALIZED_LEXICAL));
return true;
}
template <>
bool BaselineCompilerCodeGen::emitCall(JSOp op) {
MOZ_ASSERT(IsInvokeOp(op));
frame.syncStack(0);
uint32_t argc = GET_ARGC(handler.pc());
masm.move32(Imm32(argc), R0.scratchReg());
// Call IC
if (!emitNextIC()) {
return false;
}
// Update FrameInfo.
bool construct = IsConstructOp(op);
frame.popn(2 + argc + construct);
frame.push(R0);
return true;
}
template <>
bool BaselineInterpreterCodeGen::emitCall(JSOp op) {
MOZ_ASSERT(IsInvokeOp(op));
// The IC expects argc in R0.
LoadUint16Operand(masm, R0.scratchReg());
if (!emitNextIC()) {
return false;
}
// Pop the arguments. We have to reload pc/argc because the IC clobbers them.
// The return value is in R0 so we can't use that.
Register scratch = R1.scratchReg();
uint32_t extraValuesToPop = IsConstructOp(op) ? 3 : 2;
Register spReg = AsRegister(masm.getStackPointer());
LoadUint16Operand(masm, scratch);
masm.computeEffectiveAddress(
BaseValueIndex(spReg, scratch, extraValuesToPop * sizeof(Value)), spReg);
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emitSpreadCall(JSOp op) {
MOZ_ASSERT(IsInvokeOp(op));
frame.syncStack(0);
masm.move32(Imm32(1), R0.scratchReg());
// Call IC
if (!emitNextIC()) {
return false;
}
// Update FrameInfo.
bool construct = op == JSOp::SpreadNew || op == JSOp::SpreadSuperCall;
frame.popn(3 + construct);
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Call() {
return emitCall(JSOp::Call);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_CallContent() {
return emitCall(JSOp::CallContent);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_CallIgnoresRv() {
return emitCall(JSOp::CallIgnoresRv);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_CallIter() {
return emitCall(JSOp::CallIter);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_CallContentIter() {
return emitCall(JSOp::CallContentIter);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_New() {
return emitCall(JSOp::New);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_NewContent() {
return emitCall(JSOp::NewContent);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_SuperCall() {
return emitCall(JSOp::SuperCall);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Eval() {
return emitCall(JSOp::Eval);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_StrictEval() {
return emitCall(JSOp::StrictEval);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_SpreadCall() {
return emitSpreadCall(JSOp::SpreadCall);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_SpreadNew() {
return emitSpreadCall(JSOp::SpreadNew);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_SpreadSuperCall() {
return emitSpreadCall(JSOp::SpreadSuperCall);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_SpreadEval() {
return emitSpreadCall(JSOp::SpreadEval);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_StrictSpreadEval() {
return emitSpreadCall(JSOp::StrictSpreadEval);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_OptimizeSpreadCall() {
frame.popRegsAndSync(1);
if (!emitNextIC()) {
return false;
}
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_ImplicitThis() {
frame.syncStack(0);
masm.loadPtr(frame.addressOfEnvironmentChain(), R0.scratchReg());
prepareVMCall();
pushScriptNameArg(R1.scratchReg(), R2.scratchReg());
pushArg(R0.scratchReg());
using Fn = bool (*)(JSContext*, HandleObject, Handle<PropertyName*>,
MutableHandleValue);
if (!callVM<Fn, ImplicitThisOperation>()) {
return false;
}
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Instanceof() {
frame.popRegsAndSync(2);
if (!emitNextIC()) {
return false;
}
frame.push(R0, JSVAL_TYPE_BOOLEAN);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Typeof() {
frame.popRegsAndSync(1);
if (!emitNextIC()) {
return false;
}
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_TypeofExpr() {
return emit_Typeof();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_ThrowMsg() {
prepareVMCall();
pushUint8BytecodeOperandArg(R2.scratchReg());
using Fn = bool (*)(JSContext*, const unsigned);
return callVM<Fn, js::ThrowMsgOperation>();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Throw() {
// Keep value to throw in R0.
frame.popRegsAndSync(1);
prepareVMCall();
pushArg(R0);
using Fn = bool (*)(JSContext*, HandleValue);
return callVM<Fn, js::ThrowOperation>();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_ThrowWithStack() {
// Keep value to throw in R0 and the stack in R1.
frame.popRegsAndSync(2);
prepareVMCall();
pushArg(R1);
pushArg(R0);
using Fn = bool (*)(JSContext*, HandleValue, HandleValue);
return callVM<Fn, js::ThrowWithStackOperation>();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Try() {
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Finally() {
// To match the interpreter, emit an interrupt check at the start of the
// finally block.
return emitInterruptCheck();
}
static void LoadBaselineScriptResumeEntries(MacroAssembler& masm,
JSScript* script, Register dest,
Register scratch) {
MOZ_ASSERT(dest != scratch);
masm.movePtr(ImmPtr(script->jitScript()), dest);
masm.loadPtr(Address(dest, JitScript::offsetOfBaselineScript()), dest);
masm.load32(Address(dest, BaselineScript::offsetOfResumeEntriesOffset()),
scratch);
masm.addPtr(scratch, dest);
}
template <typename Handler>
void BaselineCodeGen<Handler>::emitInterpJumpToResumeEntry(Register script,
Register resumeIndex,
Register scratch) {
// Load JSScript::immutableScriptData() into |script|.
masm.loadPtr(Address(script, JSScript::offsetOfSharedData()), script);
masm.loadPtr(Address(script, SharedImmutableScriptData::offsetOfISD()),
script);
// Load the resume pcOffset in |resumeIndex|.
masm.load32(
Address(script, ImmutableScriptData::offsetOfResumeOffsetsOffset()),
scratch);
masm.computeEffectiveAddress(BaseIndex(scratch, resumeIndex, TimesFour),
scratch);
masm.load32(BaseIndex(script, scratch, TimesOne), resumeIndex);
// Add resume offset to PC, jump to it.
masm.computeEffectiveAddress(BaseIndex(script, resumeIndex, TimesOne,
ImmutableScriptData::offsetOfCode()),
script);
Address pcAddr(FramePointer, BaselineFrame::reverseOffsetOfInterpreterPC());
masm.storePtr(script, pcAddr);
emitJumpToInterpretOpLabel();
}
template <>
void BaselineCompilerCodeGen::jumpToResumeEntry(Register resumeIndex,
Register scratch1,
Register scratch2) {
LoadBaselineScriptResumeEntries(masm, handler.script(), scratch1, scratch2);
masm.loadPtr(
BaseIndex(scratch1, resumeIndex, ScaleFromElemWidth(sizeof(uintptr_t))),
scratch1);
masm.jump(scratch1);
}
template <>
void BaselineInterpreterCodeGen::jumpToResumeEntry(Register resumeIndex,
Register scratch1,
Register scratch2) {
loadScript(scratch1);
emitInterpJumpToResumeEntry(scratch1, resumeIndex, scratch2);
}
template <>
template <typename F1, typename F2>
[[nodiscard]] bool BaselineCompilerCodeGen::emitDebugInstrumentation(
const F1& ifDebuggee, const Maybe<F2>& ifNotDebuggee) {
// The JIT calls either ifDebuggee or (if present) ifNotDebuggee, because it
// knows statically whether we're compiling with debug instrumentation.
if (handler.compileDebugInstrumentation()) {
return ifDebuggee();
}
if (ifNotDebuggee) {
return (*ifNotDebuggee)();
}
return true;
}
template <>
template <typename F1, typename F2>
[[nodiscard]] bool BaselineInterpreterCodeGen::emitDebugInstrumentation(
const F1& ifDebuggee, const Maybe<F2>& ifNotDebuggee) {
// The interpreter emits both ifDebuggee and (if present) ifNotDebuggee
// paths, with a toggled jump followed by a branch on the frame's DEBUGGEE
// flag.
Label isNotDebuggee, done;
CodeOffset toggleOffset = masm.toggledJump(&isNotDebuggee);
if (!handler.addDebugInstrumentationOffset(cx, toggleOffset)) {
return false;
}
masm.branchTest32(Assembler::Zero, frame.addressOfFlags(),
Imm32(BaselineFrame::DEBUGGEE), &isNotDebuggee);
if (!ifDebuggee()) {
return false;
}
if (ifNotDebuggee) {
masm.jump(&done);
}
masm.bind(&isNotDebuggee);
if (ifNotDebuggee && !(*ifNotDebuggee)()) {
return false;
}
masm.bind(&done);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_PushLexicalEnv() {
// Call a stub to push the block on the block chain.
prepareVMCall();
masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
pushScriptGCThingArg(ScriptGCThingType::Scope, R1.scratchReg(),
R2.scratchReg());
pushArg(R0.scratchReg());
using Fn = bool (*)(JSContext*, BaselineFrame*, Handle<LexicalScope*>);
return callVM<Fn, jit::PushLexicalEnv>();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_PushClassBodyEnv() {
prepareVMCall();
masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
pushScriptGCThingArg(ScriptGCThingType::Scope, R1.scratchReg(),
R2.scratchReg());
pushArg(R0.scratchReg());
using Fn = bool (*)(JSContext*, BaselineFrame*, Handle<ClassBodyScope*>);
return callVM<Fn, jit::PushClassBodyEnv>();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_PopLexicalEnv() {
frame.syncStack(0);
Register scratch1 = R0.scratchReg();
auto ifDebuggee = [this, scratch1]() {
masm.loadBaselineFramePtr(FramePointer, scratch1);
prepareVMCall();
pushBytecodePCArg();
pushArg(scratch1);
using Fn = bool (*)(JSContext*, BaselineFrame*, const jsbytecode*);
return callVM<Fn, jit::DebugLeaveThenPopLexicalEnv>();
};
auto ifNotDebuggee = [this, scratch1]() {
Register scratch2 = R1.scratchReg();
masm.loadPtr(frame.addressOfEnvironmentChain(), scratch1);
masm.debugAssertObjectHasClass(scratch1, scratch2,
&LexicalEnvironmentObject::class_);
Address enclosingAddr(scratch1,
EnvironmentObject::offsetOfEnclosingEnvironment());
masm.unboxObject(enclosingAddr, scratch1);
masm.storePtr(scratch1, frame.addressOfEnvironmentChain());
return true;
};
return emitDebugInstrumentation(ifDebuggee, mozilla::Some(ifNotDebuggee));
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_FreshenLexicalEnv() {
frame.syncStack(0);
masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
auto ifDebuggee = [this]() {
prepareVMCall();
pushBytecodePCArg();
pushArg(R0.scratchReg());
using Fn = bool (*)(JSContext*, BaselineFrame*, const jsbytecode*);
return callVM<Fn, jit::DebuggeeFreshenLexicalEnv>();
};
auto ifNotDebuggee = [this]() {
prepareVMCall();
pushArg(R0.scratchReg());
using Fn = bool (*)(JSContext*, BaselineFrame*);
return callVM<Fn, jit::FreshenLexicalEnv>();
};
return emitDebugInstrumentation(ifDebuggee, mozilla::Some(ifNotDebuggee));
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_RecreateLexicalEnv() {
frame.syncStack(0);
masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
auto ifDebuggee = [this]() {
prepareVMCall();
pushBytecodePCArg();
pushArg(R0.scratchReg());
using Fn = bool (*)(JSContext*, BaselineFrame*, const jsbytecode*);
return callVM<Fn, jit::DebuggeeRecreateLexicalEnv>();
};
auto ifNotDebuggee = [this]() {
prepareVMCall();
pushArg(R0.scratchReg());
using Fn = bool (*)(JSContext*, BaselineFrame*);
return callVM<Fn, jit::RecreateLexicalEnv>();
};
return emitDebugInstrumentation(ifDebuggee, mozilla::Some(ifNotDebuggee));
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_DebugLeaveLexicalEnv() {
auto ifDebuggee = [this]() {
prepareVMCall();
masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
pushBytecodePCArg();
pushArg(R0.scratchReg());
using Fn = bool (*)(JSContext*, BaselineFrame*, const jsbytecode*);
return callVM<Fn, jit::DebugLeaveLexicalEnv>();
};
return emitDebugInstrumentation(ifDebuggee);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_PushVarEnv() {
prepareVMCall();
masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
pushScriptGCThingArg(ScriptGCThingType::Scope, R1.scratchReg(),
R2.scratchReg());
pushArg(R0.scratchReg());
using Fn = bool (*)(JSContext*, BaselineFrame*, Handle<Scope*>);
return callVM<Fn, jit::PushVarEnv>();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_EnterWith() {
// Pop "with" object to R0.
frame.popRegsAndSync(1);
// Call a stub to push the object onto the environment chain.
prepareVMCall();
pushScriptGCThingArg(ScriptGCThingType::Scope, R1.scratchReg(),
R2.scratchReg());
pushArg(R0);
masm.loadBaselineFramePtr(FramePointer, R1.scratchReg());
pushArg(R1.scratchReg());
using Fn =
bool (*)(JSContext*, BaselineFrame*, HandleValue, Handle<WithScope*>);
return callVM<Fn, jit::EnterWith>();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_LeaveWith() {
// Call a stub to pop the with object from the environment chain.
prepareVMCall();
masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
pushArg(R0.scratchReg());
using Fn = bool (*)(JSContext*, BaselineFrame*);
return callVM<Fn, jit::LeaveWith>();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Exception() {
prepareVMCall();
using Fn = bool (*)(JSContext*, MutableHandleValue);
if (!callVM<Fn, GetAndClearException>()) {
return false;
}
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_ExceptionAndStack() {
// First call into the VM to store the exception stack.
{
prepareVMCall();
using Fn = bool (*)(JSContext*, MutableHandleValue);
if (!callVM<Fn, GetPendingExceptionStack>()) {
return false;
}
frame.push(R0);
}
// Now get the actual exception value and clear the exception state.
{
prepareVMCall();
using Fn = bool (*)(JSContext*, MutableHandleValue);
if (!callVM<Fn, GetAndClearException>()) {
return false;
}
frame.push(R0);
}
// Finally swap the stack and the exception.
frame.popRegsAndSync(2);
frame.push(R1);
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Debugger() {
prepareVMCall();
frame.assertSyncedStack();
masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
pushArg(R0.scratchReg());
using Fn = bool (*)(JSContext*, BaselineFrame*);
if (!callVM<Fn, jit::OnDebuggerStatement>()) {
return false;
}
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emitDebugEpilogue() {
auto ifDebuggee = [this]() {
// Move return value into the frame's rval slot.
masm.storeValue(JSReturnOperand, frame.addressOfReturnValue());
masm.or32(Imm32(BaselineFrame::HAS_RVAL), frame.addressOfFlags());
// Load BaselineFrame pointer in R0.
frame.syncStack(0);
masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
prepareVMCall();
pushBytecodePCArg();
pushArg(R0.scratchReg());
const RetAddrEntry::Kind kind = RetAddrEntry::Kind::DebugEpilogue;
using Fn = bool (*)(JSContext*, BaselineFrame*, const jsbytecode*);
if (!callVM<Fn, jit::DebugEpilogueOnBaselineReturn>(kind)) {
return false;
}
masm.loadValue(frame.addressOfReturnValue(), JSReturnOperand);
return true;
};
return emitDebugInstrumentation(ifDebuggee);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emitReturn() {
if (handler.shouldEmitDebugEpilogueAtReturnOp()) {
if (!emitDebugEpilogue()) {
return false;
}
}
// Only emit the jump if this JSOp::RetRval is not the last instruction.
// Not needed for last instruction, because last instruction flows
// into return label.
if (!handler.isDefinitelyLastOp()) {
masm.jump(&return_);
}
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Return() {
frame.assertStackDepth(1);
frame.popValue(JSReturnOperand);
return emitReturn();
}
template <typename Handler>
void BaselineCodeGen<Handler>::emitLoadReturnValue(ValueOperand val) {
Label done, noRval;
masm.branchTest32(Assembler::Zero, frame.addressOfFlags(),
Imm32(BaselineFrame::HAS_RVAL), &noRval);
masm.loadValue(frame.addressOfReturnValue(), val);
masm.jump(&done);
masm.bind(&noRval);
masm.moveValue(UndefinedValue(), val);
masm.bind(&done);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_RetRval() {
frame.assertStackDepth(0);
masm.moveValue(UndefinedValue(), JSReturnOperand);
if (!handler.maybeScript() || !handler.maybeScript()->noScriptRval()) {
// Return the value in the return value slot, if any.
Label done;
Address flags = frame.addressOfFlags();
masm.branchTest32(Assembler::Zero, flags, Imm32(BaselineFrame::HAS_RVAL),
&done);
masm.loadValue(frame.addressOfReturnValue(), JSReturnOperand);
masm.bind(&done);
}
return emitReturn();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_ToPropertyKey() {
frame.popRegsAndSync(1);
if (!emitNextIC()) {
return false;
}
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_ToAsyncIter() {
frame.syncStack(0);
masm.unboxObject(frame.addressOfStackValue(-2), R0.scratchReg());
masm.loadValue(frame.addressOfStackValue(-1), R1);
prepareVMCall();
pushArg(R1);
pushArg(R0.scratchReg());
using Fn = JSObject* (*)(JSContext*, HandleObject, HandleValue);
if (!callVM<Fn, js::CreateAsyncFromSyncIterator>()) {
return false;
}
masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0);
frame.popn(2);
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_CanSkipAwait() {
frame.syncStack(0);
masm.loadValue(frame.addressOfStackValue(-1), R0);
prepareVMCall();
pushArg(R0);
using Fn = bool (*)(JSContext*, HandleValue, bool* canSkip);
if (!callVM<Fn, js::CanSkipAwait>()) {
return false;
}
masm.tagValue(JSVAL_TYPE_BOOLEAN, ReturnReg, R0);
frame.push(R0, JSVAL_TYPE_BOOLEAN);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_MaybeExtractAwaitValue() {
frame.syncStack(0);
masm.loadValue(frame.addressOfStackValue(-2), R0);
masm.unboxBoolean(frame.addressOfStackValue(-1), R1.scratchReg());
Label cantExtract;
masm.branchIfFalseBool(R1.scratchReg(), &cantExtract);
prepareVMCall();
pushArg(R0);
using Fn = bool (*)(JSContext*, HandleValue, MutableHandleValue);
if (!callVM<Fn, js::ExtractAwaitValue>()) {
return false;
}
masm.storeValue(R0, frame.addressOfStackValue(-2));
masm.bind(&cantExtract);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_AsyncAwait() {
frame.syncStack(0);
masm.loadValue(frame.addressOfStackValue(-2), R1);
masm.unboxObject(frame.addressOfStackValue(-1), R0.scratchReg());
prepareVMCall();
pushArg(R1);
pushArg(R0.scratchReg());
using Fn = JSObject* (*)(JSContext*, Handle<AsyncFunctionGeneratorObject*>,
HandleValue);
if (!callVM<Fn, js::AsyncFunctionAwait>()) {
return false;
}
masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0);
frame.popn(2);
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_AsyncResolve() {
frame.syncStack(0);
masm.loadValue(frame.addressOfStackValue(-2), R1);
masm.unboxObject(frame.addressOfStackValue(-1), R0.scratchReg());
prepareVMCall();
pushArg(R1);
pushArg(R0.scratchReg());
using Fn = JSObject* (*)(JSContext*, Handle<AsyncFunctionGeneratorObject*>,
HandleValue);
if (!callVM<Fn, js::AsyncFunctionResolve>()) {
return false;
}
masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0);
frame.popn(2);
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_AsyncReject() {
frame.syncStack(0);
masm.loadValue(frame.addressOfStackValue(-3), R2);
masm.loadValue(frame.addressOfStackValue(-2), R1);
masm.unboxObject(frame.addressOfStackValue(-1), R0.scratchReg());
prepareVMCall();
pushArg(R1);
pushArg(R2);
pushArg(R0.scratchReg());
using Fn = JSObject* (*)(JSContext*, Handle<AsyncFunctionGeneratorObject*>,
HandleValue, HandleValue);
if (!callVM<Fn, js::AsyncFunctionReject>()) {
return false;
}
masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0);
frame.popn(3);
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_CheckObjCoercible() {
frame.syncStack(0);
masm.loadValue(frame.addressOfStackValue(-1), R0);
Label fail, done;
masm.branchTestUndefined(Assembler::Equal, R0, &fail);
masm.branchTestNull(Assembler::NotEqual, R0, &done);
masm.bind(&fail);
prepareVMCall();
pushArg(R0);
using Fn = bool (*)(JSContext*, HandleValue);
if (!callVM<Fn, ThrowObjectCoercible>()) {
return false;
}
masm.bind(&done);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_ToString() {
// Keep top stack value in R0.
frame.popRegsAndSync(1);
// Inline path for string.
Label done;
masm.branchTestString(Assembler::Equal, R0, &done);
prepareVMCall();
pushArg(R0);
// Call ToStringSlow which doesn't handle string inputs.
using Fn = JSString* (*)(JSContext*, HandleValue);
if (!callVM<Fn, ToStringSlow<CanGC>>()) {
return false;
}
masm.tagValue(JSVAL_TYPE_STRING, ReturnReg, R0);
masm.bind(&done);
frame.push(R0);
return true;
}
static constexpr uint32_t TableSwitchOpLowOffset = 1 * JUMP_OFFSET_LEN;
static constexpr uint32_t TableSwitchOpHighOffset = 2 * JUMP_OFFSET_LEN;
static constexpr uint32_t TableSwitchOpFirstResumeIndexOffset =
3 * JUMP_OFFSET_LEN;
template <>
void BaselineCompilerCodeGen::emitGetTableSwitchIndex(ValueOperand val,
Register dest,
Register scratch1,
Register scratch2) {
jsbytecode* pc = handler.pc();
jsbytecode* defaultpc = pc + GET_JUMP_OFFSET(pc);
Label* defaultLabel = handler.labelOf(defaultpc);
int32_t low = GET_JUMP_OFFSET(pc + TableSwitchOpLowOffset);
int32_t high = GET_JUMP_OFFSET(pc + TableSwitchOpHighOffset);
int32_t length = high - low + 1;
// Jump to the 'default' pc if not int32 (tableswitch is only used when
// all cases are int32).
masm.branchTestInt32(Assembler::NotEqual, val, defaultLabel);
masm.unboxInt32(val, dest);
// Subtract 'low'. Bounds check.
if (low != 0) {
masm.sub32(Imm32(low), dest);
}
masm.branch32(Assembler::AboveOrEqual, dest, Imm32(length), defaultLabel);
}
template <>
void BaselineInterpreterCodeGen::emitGetTableSwitchIndex(ValueOperand val,
Register dest,
Register scratch1,
Register scratch2) {
// Jump to the 'default' pc if not int32 (tableswitch is only used when
// all cases are int32).
Label done, jumpToDefault;
masm.branchTestInt32(Assembler::NotEqual, val, &jumpToDefault);
masm.unboxInt32(val, dest);
Register pcReg = LoadBytecodePC(masm, scratch1);
Address lowAddr(pcReg, sizeof(jsbytecode) + TableSwitchOpLowOffset);
Address highAddr(pcReg, sizeof(jsbytecode) + TableSwitchOpHighOffset);
// Jump to default if val > high.
masm.branch32(Assembler::LessThan, highAddr, dest, &jumpToDefault);
// Jump to default if val < low.
masm.load32(lowAddr, scratch2);
masm.branch32(Assembler::GreaterThan, scratch2, dest, &jumpToDefault);
// index := val - low.
masm.sub32(scratch2, dest);
masm.jump(&done);
masm.bind(&jumpToDefault);
emitJump();
masm.bind(&done);
}
template <>
void BaselineCompilerCodeGen::emitTableSwitchJump(Register key,
Register scratch1,
Register scratch2) {
// Jump to resumeEntries[firstResumeIndex + key].
// Note: BytecodeEmitter::allocateResumeIndex static_asserts
// |firstResumeIndex * sizeof(uintptr_t)| fits in int32_t.
uint32_t firstResumeIndex =
GET_RESUMEINDEX(handler.pc() + TableSwitchOpFirstResumeIndexOffset);
LoadBaselineScriptResumeEntries(masm, handler.script(), scratch1, scratch2);
masm.loadPtr(BaseIndex(scratch1, key, ScaleFromElemWidth(sizeof(uintptr_t)),
firstResumeIndex * sizeof(uintptr_t)),
scratch1);
masm.jump(scratch1);
}
template <>
void BaselineInterpreterCodeGen::emitTableSwitchJump(Register key,
Register scratch1,
Register scratch2) {
// Load the op's firstResumeIndex in scratch1.
LoadUint24Operand(masm, TableSwitchOpFirstResumeIndexOffset, scratch1);
masm.add32(key, scratch1);
jumpToResumeEntry(scratch1, key, scratch2);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_TableSwitch() {
frame.popRegsAndSync(1);
Register key = R0.scratchReg();
Register scratch1 = R1.scratchReg();
Register scratch2 = R2.scratchReg();
// Call a stub to convert R0 from double to int32 if needed.
// Note: this stub may clobber scratch1.
masm.call(cx->runtime()->jitRuntime()->getDoubleToInt32ValueStub());
// Load the index in the jump table in |key|, or branch to default pc if not
// int32 or out-of-range.
emitGetTableSwitchIndex(R0, key, scratch1, scratch2);
// Jump to the target pc.
emitTableSwitchJump(key, scratch1, scratch2);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Iter() {
frame.popRegsAndSync(1);
if (!emitNextIC()) {
return false;
}
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_MoreIter() {
frame.syncStack(0);
masm.unboxObject(frame.addressOfStackValue(-1), R1.scratchReg());
masm.iteratorMore(R1.scratchReg(), R0, R2.scratchReg());
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emitIsMagicValue() {
frame.syncStack(0);
Label isMagic, done;
masm.branchTestMagic(Assembler::Equal, frame.addressOfStackValue(-1),
&isMagic);
masm.moveValue(BooleanValue(false), R0);
masm.jump(&done);
masm.bind(&isMagic);
masm.moveValue(BooleanValue(true), R0);
masm.bind(&done);
frame.push(R0, JSVAL_TYPE_BOOLEAN);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_IsNoIter() {
return emitIsMagicValue();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_EndIter() {
// Pop iterator value.
frame.pop();
// Pop the iterator object to close in R0.
frame.popRegsAndSync(1);
AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
MOZ_ASSERT(!regs.has(FramePointer));
if (HasInterpreterPCReg()) {
regs.take(InterpreterPCReg);
}
Register obj = R0.scratchReg();
regs.take(obj);
masm.unboxObject(R0, obj);
Register temp1 = regs.takeAny();
Register temp2 = regs.takeAny();
Register temp3 = regs.takeAny();
masm.iteratorClose(obj, temp1, temp2, temp3);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_CloseIter() {
frame.popRegsAndSync(1);
Register iter = R0.scratchReg();
masm.unboxObject(R0, iter);
return emitNextIC();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_OptimizeGetIterator() {
frame.popRegsAndSync(1);
if (!emitNextIC()) {
return false;
}
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_IsGenClosing() {
return emitIsMagicValue();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_IsNullOrUndefined() {
frame.syncStack(0);
Label isNullOrUndefined, done;
masm.branchTestNull(Assembler::Equal, frame.addressOfStackValue(-1),
&isNullOrUndefined);
masm.branchTestUndefined(Assembler::Equal, frame.addressOfStackValue(-1),
&isNullOrUndefined);
masm.moveValue(BooleanValue(false), R0);
masm.jump(&done);
masm.bind(&isNullOrUndefined);
masm.moveValue(BooleanValue(true), R0);
masm.bind(&done);
frame.push(R0, JSVAL_TYPE_BOOLEAN);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_GetRval() {
frame.syncStack(0);
emitLoadReturnValue(R0);
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_SetRval() {
// Store to the frame's return value slot.
frame.storeStackValue(-1, frame.addressOfReturnValue(), R2);
masm.or32(Imm32(BaselineFrame::HAS_RVAL), frame.addressOfFlags());
frame.pop();
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Callee() {
MOZ_ASSERT_IF(handler.maybeScript(), handler.maybeScript()->function());
frame.syncStack(0);
masm.loadFunctionFromCalleeToken(frame.addressOfCalleeToken(),
R0.scratchReg());
masm.tagValue(JSVAL_TYPE_OBJECT, R0.scratchReg(), R0);
frame.push(R0);
return true;
}
template <>
bool BaselineCompilerCodeGen::emit_EnvCallee() {
frame.syncStack(0);
uint8_t numHops = GET_UINT8(handler.pc());
Register scratch = R0.scratchReg();
masm.loadPtr(frame.addressOfEnvironmentChain(), scratch);
for (unsigned i = 0; i < numHops; i++) {
Address nextAddr(scratch,
EnvironmentObject::offsetOfEnclosingEnvironment());
masm.unboxObject(nextAddr, scratch);
}
masm.loadValue(Address(scratch, CallObject::offsetOfCallee()), R0);
frame.push(R0);
return true;
}
template <>
bool BaselineInterpreterCodeGen::emit_EnvCallee() {
Register scratch = R0.scratchReg();
Register env = R1.scratchReg();
static_assert(JSOpLength_EnvCallee - sizeof(jsbytecode) == ENVCOORD_HOPS_LEN,
"op must have uint8 operand for LoadAliasedVarEnv");
// Load the right environment object.
masm.loadPtr(frame.addressOfEnvironmentChain(), env);
LoadAliasedVarEnv(masm, env, scratch);
masm.pushValue(Address(env, CallObject::offsetOfCallee()));
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_SuperBase() {
frame.popRegsAndSync(1);
Register scratch = R0.scratchReg();
Register proto = R1.scratchReg();
// Unbox callee.
masm.unboxObject(R0, scratch);
// Load [[HomeObject]]
Address homeObjAddr(scratch,
FunctionExtended::offsetOfMethodHomeObjectSlot());
masm.assertFunctionIsExtended(scratch);
#ifdef DEBUG
Label isObject;
masm.branchTestObject(Assembler::Equal, homeObjAddr, &isObject);
masm.assumeUnreachable("[[HomeObject]] must be Object");
masm.bind(&isObject);
#endif
masm.unboxObject(homeObjAddr, scratch);
// Load prototype from [[HomeObject]]
masm.loadObjProto(scratch, proto);
#ifdef DEBUG
// We won't encounter a lazy proto, because the prototype is guaranteed to
// either be a JSFunction or a PlainObject, and only proxy objects can have a
// lazy proto.
MOZ_ASSERT(uintptr_t(TaggedProto::LazyProto) == 1);
Label proxyCheckDone;
masm.branchPtr(Assembler::NotEqual, proto, ImmWord(1), &proxyCheckDone);
masm.assumeUnreachable("Unexpected lazy proto in JSOp::SuperBase");
masm.bind(&proxyCheckDone);
#endif
Label nullProto, done;
masm.branchPtr(Assembler::Equal, proto, ImmWord(0), &nullProto);
// Box prototype and return
masm.tagValue(JSVAL_TYPE_OBJECT, proto, R1);
masm.jump(&done);
masm.bind(&nullProto);
masm.moveValue(NullValue(), R1);
masm.bind(&done);
frame.push(R1);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_SuperFun() {
frame.popRegsAndSync(1);
Register callee = R0.scratchReg();
Register proto = R1.scratchReg();
#ifdef DEBUG
Register scratch = R2.scratchReg();
#endif
// Unbox callee.
masm.unboxObject(R0, callee);
#ifdef DEBUG
Label classCheckDone;
masm.branchTestObjIsFunction(Assembler::Equal, callee, scratch, callee,
&classCheckDone);
masm.assumeUnreachable("Unexpected non-JSFunction callee in JSOp::SuperFun");
masm.bind(&classCheckDone);
#endif
// Load prototype of callee
masm.loadObjProto(callee, proto);
#ifdef DEBUG
// We won't encounter a lazy proto, because |callee| is guaranteed to be a
// JSFunction and only proxy objects can have a lazy proto.
MOZ_ASSERT(uintptr_t(TaggedProto::LazyProto) == 1);
Label proxyCheckDone;
masm.branchPtr(Assembler::NotEqual, proto, ImmWord(1), &proxyCheckDone);
masm.assumeUnreachable("Unexpected lazy proto in JSOp::SuperFun");
masm.bind(&proxyCheckDone);
#endif
Label nullProto, done;
masm.branchPtr(Assembler::Equal, proto, ImmWord(0), &nullProto);
// Box prototype and return
masm.tagValue(JSVAL_TYPE_OBJECT, proto, R1);
masm.jump(&done);
masm.bind(&nullProto);
masm.moveValue(NullValue(), R1);
masm.bind(&done);
frame.push(R1);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Arguments() {
frame.syncStack(0);
MOZ_ASSERT_IF(handler.maybeScript(), handler.maybeScript()->needsArgsObj());
prepareVMCall();
masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
pushArg(R0.scratchReg());
using Fn = bool (*)(JSContext*, BaselineFrame*, MutableHandleValue);
if (!callVM<Fn, jit::NewArgumentsObject>()) {
return false;
}
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Rest() {
frame.syncStack(0);
if (!emitNextIC()) {
return false;
}
// Mark R0 as pushed stack value.
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Generator() {
frame.assertStackDepth(0);
masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
prepareVMCall();
pushArg(R0.scratchReg());
using Fn = JSObject* (*)(JSContext*, BaselineFrame*);
if (!callVM<Fn, jit::CreateGeneratorFromFrame>()) {
return false;
}
masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0);
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emitSuspend(JSOp op) {
MOZ_ASSERT(op == JSOp::InitialYield || op == JSOp::Yield ||
op == JSOp::Await);
// Load the generator object in R2, but leave the return value on the
// expression stack.
Register genObj = R2.scratchReg();
if (op == JSOp::InitialYield) {
// Generator and return value are one and the same.
frame.syncStack(0);
frame.assertStackDepth(1);
masm.unboxObject(frame.addressOfStackValue(-1), genObj);
} else {
frame.popRegsAndSync(1);
masm.unboxObject(R0, genObj);
}
if (frame.hasKnownStackDepth(1) && !handler.canHaveFixedSlots()) {
// If the expression stack is empty, we can inline the Yield. Note that this
// branch is never taken for the interpreter because it doesn't know static
// stack depths.
MOZ_ASSERT_IF(op == JSOp::InitialYield && handler.maybePC(),
GET_RESUMEINDEX(handler.maybePC()) == 0);
Address resumeIndexSlot(genObj,
AbstractGeneratorObject::offsetOfResumeIndexSlot());
Register temp = R1.scratchReg();
if (op == JSOp::InitialYield) {
masm.storeValue(Int32Value(0), resumeIndexSlot);
} else {
jsbytecode* pc = handler.maybePC();
MOZ_ASSERT(pc, "compiler-only code never has a null pc");
masm.move32(Imm32(GET_RESUMEINDEX(pc)), temp);
masm.storeValue(JSVAL_TYPE_INT32, temp, resumeIndexSlot);
}
Register envObj = R0.scratchReg();
Address envChainSlot(
genObj, AbstractGeneratorObject::offsetOfEnvironmentChainSlot());
masm.loadPtr(frame.addressOfEnvironmentChain(), envObj);
masm.guardedCallPreBarrierAnyZone(envChainSlot, MIRType::Value, temp);
masm.storeValue(JSVAL_TYPE_OBJECT, envObj, envChainSlot);
Label skipBarrier;
masm.branchPtrInNurseryChunk(Assembler::Equal, genObj, temp, &skipBarrier);
masm.branchPtrInNurseryChunk(Assembler::NotEqual, envObj, temp,
&skipBarrier);
MOZ_ASSERT(genObj == R2.scratchReg());
masm.call(&postBarrierSlot_);
masm.bind(&skipBarrier);
} else {
masm.loadBaselineFramePtr(FramePointer, R1.scratchReg());
computeFrameSize(R0.scratchReg());
prepareVMCall();
pushBytecodePCArg();
pushArg(R0.scratchReg());
pushArg(R1.scratchReg());
pushArg(genObj);
using Fn = bool (*)(JSContext*, HandleObject, BaselineFrame*, uint32_t,
const jsbytecode*);
if (!callVM<Fn, jit::NormalSuspend>()) {
return false;
}
}
masm.loadValue(frame.addressOfStackValue(-1), JSReturnOperand);
if (!emitReturn()) {
return false;
}
// Three values are pushed onto the stack when resuming the generator,
// replacing the one slot that holds the return value.
frame.incStackDepth(2);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_InitialYield() {
return emitSuspend(JSOp::InitialYield);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Yield() {
return emitSuspend(JSOp::Yield);
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Await() {
return emitSuspend(JSOp::Await);
}
template <>
template <typename F>
bool BaselineCompilerCodeGen::emitAfterYieldDebugInstrumentation(
const F& ifDebuggee, Register) {
if (handler.compileDebugInstrumentation()) {
return ifDebuggee();
}
return true;
}
template <>
template <typename F>
bool BaselineInterpreterCodeGen::emitAfterYieldDebugInstrumentation(
const F& ifDebuggee, Register scratch) {
// Note that we can't use emitDebugInstrumentation here because the frame's
// DEBUGGEE flag hasn't been initialized yet.
// If the current Realm is not a debuggee we're done.
Label done;
CodeOffset toggleOffset = masm.toggledJump(&done);
if (!handler.addDebugInstrumentationOffset(cx, toggleOffset)) {
return false;
}
masm.loadPtr(AbsoluteAddress(cx->addressOfRealm()), scratch);
masm.branchTest32(Assembler::Zero,
Address(scratch, Realm::offsetOfDebugModeBits()),
Imm32(Realm::debugModeIsDebuggeeBit()), &done);
if (!ifDebuggee()) {
return false;
}
masm.bind(&done);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_AfterYield() {
if (!emit_JumpTarget()) {
return false;
}
auto ifDebuggee = [this]() {
frame.assertSyncedStack();
masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
prepareVMCall();
pushArg(R0.scratchReg());
const RetAddrEntry::Kind kind = RetAddrEntry::Kind::DebugAfterYield;
using Fn = bool (*)(JSContext*, BaselineFrame*);
if (!callVM<Fn, jit::DebugAfterYield>(kind)) {
return false;
}
return true;
};
return emitAfterYieldDebugInstrumentation(ifDebuggee, R0.scratchReg());
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_FinalYieldRval() {
// Store generator in R0.
frame.popRegsAndSync(1);
masm.unboxObject(R0, R0.scratchReg());
prepareVMCall();
pushBytecodePCArg();
pushArg(R0.scratchReg());
using Fn = bool (*)(JSContext*, HandleObject, const jsbytecode*);
if (!callVM<Fn, jit::FinalSuspend>()) {
return false;
}
masm.loadValue(frame.addressOfReturnValue(), JSReturnOperand);
return emitReturn();
}
template <>
void BaselineCompilerCodeGen::emitJumpToInterpretOpLabel() {
TrampolinePtr code =
cx->runtime()->jitRuntime()->baselineInterpreter().interpretOpAddr();
masm.jump(code);
}
template <>
void BaselineInterpreterCodeGen::emitJumpToInterpretOpLabel() {
masm.jump(handler.interpretOpLabel());
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emitEnterGeneratorCode(Register script,
Register resumeIndex,
Register scratch) {
// Resume in either the BaselineScript (if present) or Baseline Interpreter.
static_assert(BaselineDisabledScript == 0x1,
"Comparison below requires specific sentinel encoding");
// Initialize the icScript slot in the baseline frame.
masm.loadJitScript(script, scratch);
masm.computeEffectiveAddress(Address(scratch, JitScript::offsetOfICScript()),
scratch);
Address icScriptAddr(FramePointer, BaselineFrame::reverseOffsetOfICScript());
masm.storePtr(scratch, icScriptAddr);
Label noBaselineScript;
masm.loadJitScript(script, scratch);
masm.loadPtr(Address(scratch, JitScript::offsetOfBaselineScript()), scratch);
masm.branchPtr(Assembler::BelowOrEqual, scratch,
ImmPtr(BaselineDisabledScriptPtr), &noBaselineScript);
masm.load32(Address(scratch, BaselineScript::offsetOfResumeEntriesOffset()),
script);
masm.addPtr(scratch, script);
masm.loadPtr(
BaseIndex(script, resumeIndex, ScaleFromElemWidth(sizeof(uintptr_t))),
scratch);
masm.jump(scratch);
masm.bind(&noBaselineScript);
// Initialize interpreter frame fields.
Address flagsAddr(FramePointer, BaselineFrame::reverseOffsetOfFlags());
Address scriptAddr(FramePointer,
BaselineFrame::reverseOffsetOfInterpreterScript());
masm.or32(Imm32(BaselineFrame::RUNNING_IN_INTERPRETER), flagsAddr);
masm.storePtr(script, scriptAddr);
// Initialize pc and jump to it.
emitInterpJumpToResumeEntry(script, resumeIndex, scratch);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_Resume() {
frame.syncStack(0);
masm.assertStackAlignment(sizeof(Value), 0);
AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
MOZ_ASSERT(!regs.has(FramePointer));
if (HasInterpreterPCReg()) {
regs.take(InterpreterPCReg);
}
saveInterpreterPCReg();
// Load generator object.
Register genObj = regs.takeAny();
masm.unboxObject(frame.addressOfStackValue(-3), genObj);
// Load callee.
Register callee = regs.takeAny();
masm.unboxObject(
Address(genObj, AbstractGeneratorObject::offsetOfCalleeSlot()), callee);
// Save a pointer to the JSOp::Resume operand stack Values.
Register callerStackPtr = regs.takeAny();
masm.computeEffectiveAddress(frame.addressOfStackValue(-1), callerStackPtr);
// Branch to |interpret| to resume the generator in the C++ interpreter if the
// script does not have a JitScript.
Label interpret;
Register scratch1 = regs.takeAny();
masm.loadPrivate(Address(callee, JSFunction::offsetOfJitInfoOrScript()),
scratch1);
masm.branchIfScriptHasNoJitScript(scratch1, &interpret);
// Push |undefined| for all formals.
Register scratch2 = regs.takeAny();
Label loop, loopDone;
masm.loadFunctionArgCount(callee, scratch2);
static_assert(sizeof(Value) == 8);
#ifndef JS_CODEGEN_NONE
static_assert(JitStackAlignment == 16 || JitStackAlignment == 8);
#endif
// If JitStackValueAlignment == 1, then we were already correctly aligned on
// entry, as guaranteed by the assertStackAlignment at the entry to this
// function.
if (JitStackValueAlignment > 1) {
Register alignment = regs.takeAny();
masm.moveStackPtrTo(alignment);
masm.alignJitStackBasedOnNArgs(scratch2, false);
// Compute alignment adjustment.
masm.subStackPtrFrom(alignment);
// Some code, like BaselineFrame::trace, will inspect the whole range of
// the stack frame. In order to ensure that garbage data left behind from
// previous activations doesn't confuse other machinery, we zero out the
// alignment bytes.
Label alignmentZero;
masm.branchPtr(Assembler::Equal, alignment, ImmWord(0), &alignmentZero);
// Since we know prior to the stack alignment that the stack was 8 byte
// aligned, and JitStackAlignment is 8 or 16 bytes, if we are doing an
// alignment then we -must- have aligned by subtracting 8 bytes from
// the stack pointer.
//
// So we can freely store a valid double here.
masm.storeValue(DoubleValue(0), Address(masm.getStackPointer(), 0));
masm.bind(&alignmentZero);
}
masm.branchTest32(Assembler::Zero, scratch2, scratch2, &loopDone);
masm.bind(&loop);
{
masm.pushValue(UndefinedValue());
masm.branchSub32(Assembler::NonZero, Imm32(1), scratch2, &loop);
}
masm.bind(&loopDone);
// Push |undefined| for |this|.
masm.pushValue(UndefinedValue());
#ifdef DEBUG
// Update BaselineFrame debugFrameSize field.
masm.mov(FramePointer, scratch2);
masm.subStackPtrFrom(scratch2);
masm.store32(scratch2, frame.addressOfDebugFrameSize());
#endif
masm.PushCalleeToken(callee, /* constructing = */ false);
masm.pushFrameDescriptorForJitCall(FrameType::BaselineJS, /* argc = */ 0);
// PushCalleeToken bumped framePushed. Reset it.
MOZ_ASSERT(masm.framePushed() == sizeof(uintptr_t));
masm.setFramePushed(0);
regs.add(callee);
// Push a fake return address on the stack. We will resume here when the
// generator returns.
Label genStart, returnTarget;
#ifdef JS_USE_LINK_REGISTER
masm.call(&genStart);
#else
masm.callAndPushReturnAddress(&genStart);
#endif
// Record the return address so the return offset -> pc mapping works.
if (!handler.recordCallRetAddr(cx, RetAddrEntry::Kind::IC,
masm.currentOffset())) {
return false;
}
masm.jump(&returnTarget);
masm.bind(&genStart);
#ifdef JS_USE_LINK_REGISTER
masm.pushReturnAddress();
#endif
// Construct BaselineFrame.
masm.push(FramePointer);
masm.moveStackPtrTo(FramePointer);
// If profiler instrumentation is on, update lastProfilingFrame on
// current JitActivation
{
Register scratchReg = scratch2;
Label skip;
AbsoluteAddress addressOfEnabled(
cx->runtime()->geckoProfiler().addressOfEnabled());
masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0), &skip);
masm.loadJSContext(scratchReg);
masm.loadPtr(Address(scratchReg, JSContext::offsetOfProfilingActivation()),
scratchReg);
masm.storePtr(
FramePointer,
Address(scratchReg, JitActivation::offsetOfLastProfilingFrame()));
masm.bind(&skip);
}
masm.subFromStackPtr(Imm32(BaselineFrame::Size()));
masm.assertStackAlignment(sizeof(Value), 0);
// Store flags and env chain.
masm.store32(Imm32(BaselineFrame::HAS_INITIAL_ENV), frame.addressOfFlags());
masm.unboxObject(
Address(genObj, AbstractGeneratorObject::offsetOfEnvironmentChainSlot()),
scratch2);
masm.storePtr(scratch2, frame.addressOfEnvironmentChain());
// Store the arguments object if there is one.
Label noArgsObj;
Address argsObjSlot(genObj, AbstractGeneratorObject::offsetOfArgsObjSlot());
masm.fallibleUnboxObject(argsObjSlot, scratch2, &noArgsObj);
{
masm.storePtr(scratch2, frame.addressOfArgsObj());
masm.or32(Imm32(BaselineFrame::HAS_ARGS_OBJ), frame.addressOfFlags());
}
masm.bind(&noArgsObj);
// Push locals and expression slots if needed.
Label noStackStorage;
Address stackStorageSlot(genObj,
AbstractGeneratorObject::offsetOfStackStorageSlot());
masm.fallibleUnboxObject(stackStorageSlot, scratch2, &noStackStorage);
{
Register initLength = regs.takeAny();
masm.loadPtr(Address(scratch2, NativeObject::offsetOfElements()), scratch2);
masm.load32(Address(scratch2, ObjectElements::offsetOfInitializedLength()),
initLength);
masm.store32(
Imm32(0),
Address(scratch2, ObjectElements::offsetOfInitializedLength()));
Label loop, loopDone;
masm.branchTest32(Assembler::Zero, initLength, initLength, &loopDone);
masm.bind(&loop);
{
masm.pushValue(Address(scratch2, 0));
masm.guardedCallPreBarrierAnyZone(Address(scratch2, 0), MIRType::Value,
scratch1);
masm.addPtr(Imm32(sizeof(Value)), scratch2);
masm.branchSub32(Assembler::NonZero, Imm32(1), initLength, &loop);
}
masm.bind(&loopDone);
regs.add(initLength);
}
masm.bind(&noStackStorage);
// Push arg, generator, resumeKind stack Values, in that order.
masm.pushValue(Address(callerStackPtr, sizeof(Value)));
masm.pushValue(JSVAL_TYPE_OBJECT, genObj);
masm.pushValue(Address(callerStackPtr, 0));
masm.switchToObjectRealm(genObj, scratch2);
// Load script in scratch1.
masm.unboxObject(
Address(genObj, AbstractGeneratorObject::offsetOfCalleeSlot()), scratch1);
masm.loadPrivate(Address(scratch1, JSFunction::offsetOfJitInfoOrScript()),
scratch1);
// Load resume index in scratch2 and mark generator as running.
Address resumeIndexSlot(genObj,
AbstractGeneratorObject::offsetOfResumeIndexSlot());
masm.unboxInt32(resumeIndexSlot, scratch2);
masm.storeValue(Int32Value(AbstractGeneratorObject::RESUME_INDEX_RUNNING),
resumeIndexSlot);
if (!emitEnterGeneratorCode(scratch1, scratch2, regs.getAny())) {
return false;
}
// Call into the VM to resume the generator in the C++ interpreter if there's
// no JitScript.
masm.bind(&interpret);
prepareVMCall();
pushArg(callerStackPtr);
pushArg(genObj);
using Fn = bool (*)(JSContext*, HandleObject, Value*, MutableHandleValue);
if (!callVM<Fn, jit::InterpretResume>()) {
return false;
}
masm.bind(&returnTarget);
// Restore Stack pointer
masm.computeEffectiveAddress(frame.addressOfStackValue(-1),
masm.getStackPointer());
// After the generator returns, we restore the stack pointer, switch back to
// the current realm, push the return value, and we're done.
if (JSScript* script = handler.maybeScript()) {
masm.switchToRealm(script->realm(), R2.scratchReg());
} else {
masm.switchToBaselineFrameRealm(R2.scratchReg());
}
restoreInterpreterPCReg();
frame.popn(3);
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_CheckResumeKind() {
// Load resumeKind in R1, generator in R0.
frame.popRegsAndSync(2);
#ifdef DEBUG
Label ok;
masm.branchTestInt32(Assembler::Equal, R1, &ok);
masm.assumeUnreachable("Expected int32 resumeKind");
masm.bind(&ok);
#endif
// If resumeKind is 'next' we don't have to do anything.
Label done;
masm.unboxInt32(R1, R1.scratchReg());
masm.branch32(Assembler::Equal, R1.scratchReg(),
Imm32(int32_t(GeneratorResumeKind::Next)), &done);
prepareVMCall();
pushArg(R1.scratchReg()); // resumeKind
masm.loadValue(frame.addressOfStackValue(-1), R2);
pushArg(R2); // arg
masm.unboxObject(R0, R0.scratchReg());
pushArg(R0.scratchReg()); // genObj
masm.loadBaselineFramePtr(FramePointer, R2.scratchReg());
pushArg(R2.scratchReg()); // frame
using Fn = bool (*)(JSContext*, BaselineFrame*,
Handle<AbstractGeneratorObject*>, HandleValue, int32_t);
if (!callVM<Fn, jit::GeneratorThrowOrReturn>()) {
return false;
}
masm.bind(&done);
return true;
}
template <>
bool BaselineCompilerCodeGen::emit_ResumeKind() {
GeneratorResumeKind resumeKind = ResumeKindFromPC(handler.pc());
frame.push(Int32Value(int32_t(resumeKind)));
return true;
}
template <>
bool BaselineInterpreterCodeGen::emit_ResumeKind() {
LoadUint8Operand(masm, R0.scratchReg());
masm.tagValue(JSVAL_TYPE_INT32, R0.scratchReg(), R0);
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_DebugCheckSelfHosted() {
#ifdef DEBUG
frame.syncStack(0);
masm.loadValue(frame.addressOfStackValue(-1), R0);
prepareVMCall();
pushArg(R0);
using Fn = bool (*)(JSContext*, HandleValue);
if (!callVM<Fn, js::Debug_CheckSelfHosted>()) {
return false;
}
#endif
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_IsConstructing() {
frame.push(MagicValue(JS_IS_CONSTRUCTING));
return true;
}
template <>
bool BaselineCompilerCodeGen::emit_JumpTarget() {
MaybeIncrementCodeCoverageCounter(masm, handler.script(), handler.pc());
return true;
}
template <>
bool BaselineInterpreterCodeGen::emit_JumpTarget() {
Register scratch1 = R0.scratchReg();
Register scratch2 = R1.scratchReg();
Label skipCoverage;
CodeOffset toggleOffset = masm.toggledJump(&skipCoverage);
masm.call(handler.codeCoverageAtPCLabel());
masm.bind(&skipCoverage);
if (!handler.codeCoverageOffsets().append(toggleOffset.offset())) {
return false;
}
// Load icIndex in scratch1.
LoadInt32Operand(masm, scratch1);
// Compute ICEntry* and store to frame->interpreterICEntry.
masm.loadPtr(frame.addressOfICScript(), scratch2);
static_assert(sizeof(ICEntry) == sizeof(uintptr_t));
masm.computeEffectiveAddress(BaseIndex(scratch2, scratch1, ScalePointer,
ICScript::offsetOfICEntries()),
scratch2);
masm.storePtr(scratch2, frame.addressOfInterpreterICEntry());
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_CheckClassHeritage() {
frame.syncStack(0);
// Leave the heritage value on the stack.
masm.loadValue(frame.addressOfStackValue(-1), R0);
prepareVMCall();
pushArg(R0);
using Fn = bool (*)(JSContext*, HandleValue);
return callVM<Fn, js::CheckClassHeritageOperation>();
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_InitHomeObject() {
// Load HomeObject in R0.
frame.popRegsAndSync(1);
// Load function off stack
Register func = R2.scratchReg();
masm.unboxObject(frame.addressOfStackValue(-1), func);
masm.assertFunctionIsExtended(func);
// Set HOMEOBJECT_SLOT
Register temp = R1.scratchReg();
Address addr(func, FunctionExtended::offsetOfMethodHomeObjectSlot());
masm.guardedCallPreBarrierAnyZone(addr, MIRType::Value, temp);
masm.storeValue(R0, addr);
Label skipBarrier;
masm.branchPtrInNurseryChunk(Assembler::Equal, func, temp, &skipBarrier);
masm.branchValueIsNurseryCell(Assembler::NotEqual, R0, temp, &skipBarrier);
masm.call(&postBarrierSlot_);
masm.bind(&skipBarrier);
return true;
}
template <>
bool BaselineCompilerCodeGen::emit_BuiltinObject() {
// Built-in objects are constants for a given global.
auto kind = BuiltinObjectKind(GET_UINT8(handler.pc()));
JSObject* builtin = BuiltinObjectOperation(cx, kind);
if (!builtin) {
return false;
}
frame.push(ObjectValue(*builtin));
return true;
}
template <>
bool BaselineInterpreterCodeGen::emit_BuiltinObject() {
prepareVMCall();
pushUint8BytecodeOperandArg(R0.scratchReg());
using Fn = JSObject* (*)(JSContext*, BuiltinObjectKind);
if (!callVM<Fn, BuiltinObjectOperation>()) {
return false;
}
masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0);
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_ObjWithProto() {
frame.syncStack(0);
// Leave the proto value on the stack for the decompiler
masm.loadValue(frame.addressOfStackValue(-1), R0);
prepareVMCall();
pushArg(R0);
using Fn = PlainObject* (*)(JSContext*, HandleValue);
if (!callVM<Fn, js::ObjectWithProtoOperation>()) {
return false;
}
masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0);
frame.pop();
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_FunWithProto() {
frame.popRegsAndSync(1);
masm.unboxObject(R0, R0.scratchReg());
masm.loadPtr(frame.addressOfEnvironmentChain(), R1.scratchReg());
prepareVMCall();
pushArg(R0.scratchReg());
pushArg(R1.scratchReg());
pushScriptGCThingArg(ScriptGCThingType::Function, R0.scratchReg(),
R1.scratchReg());
using Fn =
JSObject* (*)(JSContext*, HandleFunction, HandleObject, HandleObject);
if (!callVM<Fn, js::FunWithProtoOperation>()) {
return false;
}
masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0);
frame.push(R0);
return true;
}
template <>
bool BaselineCompilerCodeGen::emit_ImportMeta() {
// Note: this is like the interpreter implementation, but optimized a bit by
// calling GetModuleObjectForScript at compile-time.
Rooted<ModuleObject*> module(cx, GetModuleObjectForScript(handler.script()));
frame.syncStack(0);
prepareVMCall();
pushArg(ImmGCPtr(module));
using Fn = JSObject* (*)(JSContext*, HandleObject);
if (!callVM<Fn, js::GetOrCreateModuleMetaObject>()) {
return false;
}
masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0);
frame.push(R0);
return true;
}
template <>
bool BaselineInterpreterCodeGen::emit_ImportMeta() {
prepareVMCall();
pushScriptArg();
using Fn = JSObject* (*)(JSContext*, HandleScript);
if (!callVM<Fn, ImportMetaOperation>()) {
return false;
}
masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0);
frame.push(R0);
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emit_DynamicImport() {
// Put specifier into R0 and object value into R1
frame.popRegsAndSync(2);
prepareVMCall();
pushArg(R1);
pushArg(R0);
pushScriptArg();
using Fn = JSObject* (*)(JSContext*, HandleScript, HandleValue, HandleValue);
if (!callVM<Fn, js::StartDynamicModuleImport>()) {
return false;
}
masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0);
frame.push(R0);
return true;
}
template <>
bool BaselineCompilerCodeGen::emit_ForceInterpreter() {
// Caller is responsible for checking script->hasForceInterpreterOp().
MOZ_CRASH("JSOp::ForceInterpreter in baseline");
}
template <>
bool BaselineInterpreterCodeGen::emit_ForceInterpreter() {
masm.assumeUnreachable("JSOp::ForceInterpreter");
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emitPrologue() {
AutoCreatedBy acb(masm, "BaselineCodeGen<Handler>::emitPrologue");
#ifdef JS_USE_LINK_REGISTER
// Push link register from generateEnterJIT()'s BLR.
masm.pushReturnAddress();
#endif
masm.push(FramePointer);
masm.moveStackPtrTo(FramePointer);
masm.checkStackAlignment();
emitProfilerEnterFrame();
masm.subFromStackPtr(Imm32(BaselineFrame::Size()));
// Initialize BaselineFrame. Also handles env chain pre-initialization (in
// case GC gets run during stack check). For global and eval scripts, the env
// chain is in R1. For function scripts, the env chain is in the callee.
emitInitFrameFields(R1.scratchReg());
// When compiling with Debugger instrumentation, set the debuggeeness of
// the frame before any operation that can call into the VM.
if (!emitIsDebuggeeCheck()) {
return false;
}
// Initialize the env chain before any operation that may call into the VM and
// trigger a GC.
if (!initEnvironmentChain()) {
return false;
}
// Check for overrecursion before initializing locals.
if (!emitStackCheck()) {
return false;
}
emitInitializeLocals();
// Ion prologue bailouts will enter here in the Baseline Interpreter.
masm.bind(&bailoutPrologue_);
frame.assertSyncedStack();
if (JSScript* script = handler.maybeScript()) {
masm.debugAssertContextRealm(script->realm(), R1.scratchReg());
}
if (!emitDebugPrologue()) {
return false;
}
if (!emitHandleCodeCoverageAtPrologue()) {
return false;
}
if (!emitWarmUpCounterIncrement()) {
return false;
}
warmUpCheckPrologueOffset_ = CodeOffset(masm.currentOffset());
return true;
}
template <typename Handler>
bool BaselineCodeGen<Handler>::emitEpilogue() {
AutoCreatedBy acb(masm, "BaselineCodeGen<Handler>::emitEpilogue");
masm.bind(&return_);
if (!handler.shouldEmitDebugEpilogueAtReturnOp()) {
if (!emitDebugEpilogue()) {
return false;
}
}
emitProfilerExitFrame();
masm.moveToStackPtr(FramePointer);
masm.pop(FramePointer);
masm.ret();
return true;
}
MethodStatus BaselineCompiler::emitBody() {
AutoCreatedBy acb(masm, "BaselineCompiler::emitBody");
JSScript* script = handler.script();
MOZ_ASSERT(handler.pc() == script->code());
mozilla::DebugOnly<jsbytecode*> prevpc = handler.pc();
while (true) {
JSOp op = JSOp(*handler.pc());
JitSpew(JitSpew_BaselineOp, "Compiling op @ %d: %s",
int(script->pcToOffset(handler.pc())), CodeName(op));
BytecodeInfo* info = handler.analysis().maybeInfo(handler.pc());
// Skip unreachable ops.
if (!info) {
// Test if last instructions and stop emitting in that case.
handler.moveToNextPC();
if (handler.pc() >= script->codeEnd()) {
break;
}
prevpc = handler.pc();
continue;
}
if (info->jumpTarget) {
// Fully sync the stack if there are incoming jumps.
frame.syncStack(0);
frame.setStackDepth(info->stackDepth);
masm.bind(handler.labelOf(handler.pc()));
} else if (MOZ_UNLIKELY(compileDebugInstrumentation())) {
// Also fully sync the stack if the debugger is enabled.
frame.syncStack(0);
} else {
// At the beginning of any op, at most the top 2 stack-values are
// unsynced.
if (frame.stackDepth() > 2) {
frame.syncStack(2);
}
}
frame.assertValidState(*info);
// If the script has a resume offset for this pc we need to keep track of
// the native code offset.
if (info->hasResumeOffset) {
frame.assertSyncedStack();
uint32_t pcOffset = script->pcToOffset(handler.pc());
uint32_t nativeOffset = masm.currentOffset();
if (!resumeOffsetEntries_.emplaceBack(pcOffset, nativeOffset)) {
ReportOutOfMemory(cx);
return Method_Error;
}
}
// Emit traps for breakpoints and step mode.
if (MOZ_UNLIKELY(compileDebugInstrumentation()) && !emitDebugTrap()) {
return Method_Error;
}
perfSpewer_.recordInstruction(cx, masm, handler.pc(), frame);
#define EMIT_OP(OP, ...) \
case JSOp::OP: { \
AutoCreatedBy acb(masm, "op=" #OP); \
if (MOZ_UNLIKELY(!this->emit_##OP())) return Method_Error; \
} break;
switch (op) {
FOR_EACH_OPCODE(EMIT_OP)
default:
MOZ_CRASH("Unexpected op");
}
#undef EMIT_OP
MOZ_ASSERT(masm.framePushed() == 0);
// Test if last instructions and stop emitting in that case.
handler.moveToNextPC();
if (handler.pc() >= script->codeEnd()) {
break;
}
#ifdef DEBUG
prevpc = handler.pc();
#endif
}
MOZ_ASSERT(JSOp(*prevpc) == JSOp::RetRval || JSOp(*prevpc) == JSOp::Return);
return Method_Compiled;
}
bool BaselineInterpreterGenerator::emitDebugTrap() {
CodeOffset offset = masm.nopPatchableToCall();
if (!debugTrapOffsets_.append(offset.offset())) {
ReportOutOfMemory(cx);
return false;
}
return true;
}
// Register holding the bytecode pc during dispatch. This exists so the debug
// trap handler can reload the pc into this register when it's done.
static constexpr Register InterpreterPCRegAtDispatch =
HasInterpreterPCReg() ? InterpreterPCReg : R0.scratchReg();
bool BaselineInterpreterGenerator::emitInterpreterLoop() {
AutoCreatedBy acb(masm, "BaselineInterpreterGenerator::emitInterpreterLoop");
Register scratch1 = R0.scratchReg();
Register scratch2 = R1.scratchReg();
// Entry point for interpreting a bytecode op. No registers are live except
// for InterpreterPCReg.
masm.bind(handler.interpretOpWithPCRegLabel());
// Emit a patchable call for debugger breakpoints/stepping.
if (!emitDebugTrap()) {
return false;
}
Label interpretOpAfterDebugTrap;
masm.bind(&interpretOpAfterDebugTrap);
// Load pc, bytecode op.
Register pcReg = LoadBytecodePC(masm, scratch1);
masm.load8ZeroExtend(Address(pcReg, 0), scratch1);
// Jump to table[op].
{
CodeOffset label = masm.moveNearAddressWithPatch(scratch2);
if (!tableLabels_.append(label)) {
return false;
}
BaseIndex pointer(scratch2, scratch1, ScalePointer);
masm.branchToComputedAddress(pointer);
}
// At the end of each op, emit code to bump the pc and jump to the
// next op (this is also known as a threaded interpreter).
auto opEpilogue = [&](JSOp op, size_t opLength) -> bool {
MOZ_ASSERT(masm.framePushed() == 0);
if (!BytecodeFallsThrough(op)) {
// Nothing to do.
masm.assumeUnreachable("unexpected fall through");
return true;
}
// Bump frame->interpreterICEntry if needed.
if (BytecodeOpHasIC(op)) {
frame.bumpInterpreterICEntry();
}
// Bump bytecode PC.
if (HasInterpreterPCReg()) {
MOZ_ASSERT(InterpreterPCRegAtDispatch == InterpreterPCReg);
masm.addPtr(Imm32(opLength), InterpreterPCReg);
} else {
MOZ_ASSERT(InterpreterPCRegAtDispatch == scratch1);
masm.loadPtr(frame.addressOfInterpreterPC(), InterpreterPCRegAtDispatch);
masm.addPtr(Imm32(opLength), InterpreterPCRegAtDispatch);
masm.storePtr(InterpreterPCRegAtDispatch, frame.addressOfInterpreterPC());
}
if (!emitDebugTrap()) {
return false;
}
// Load the opcode, jump to table[op].
masm.load8ZeroExtend(Address(InterpreterPCRegAtDispatch, 0), scratch1);
CodeOffset label = masm.moveNearAddressWithPatch(scratch2);
if (!tableLabels_.append(label)) {
return false;
}
BaseIndex pointer(scratch2, scratch1, ScalePointer);
masm.branchToComputedAddress(pointer);
return true;
};
// Emit code for each bytecode op.
Label opLabels[JSOP_LIMIT];
#define EMIT_OP(OP, ...) \
{ \
AutoCreatedBy acb(masm, "op=" #OP); \
perfSpewer_.recordOffset(masm, JSOp::OP); \
masm.bind(&opLabels[uint8_t(JSOp::OP)]); \
handler.setCurrentOp(JSOp::OP); \
if (!this->emit_##OP()) { \
return false; \
} \
if (!opEpilogue(JSOp::OP, JSOpLength_##OP)) { \
return false; \
} \
handler.resetCurrentOp(); \
}
FOR_EACH_OPCODE(EMIT_OP)
#undef EMIT_OP
// External entry point to start interpreting bytecode ops. This is used for
// things like exception handling and OSR. DebugModeOSR patches JIT frames to
// return here from the DebugTrapHandler.
masm.bind(handler.interpretOpLabel());
interpretOpOffset_ = masm.currentOffset();
restoreInterpreterPCReg();
masm.jump(handler.interpretOpWithPCRegLabel());
// Second external entry point: this skips the debug trap for the first op
// and is used by OSR.
interpretOpNoDebugTrapOffset_ = masm.currentOffset();
restoreInterpreterPCReg();
masm.jump(&interpretOpAfterDebugTrap);
// External entry point for Ion prologue bailouts.
bailoutPrologueOffset_ = CodeOffset(masm.currentOffset());
restoreInterpreterPCReg();
masm.jump(&bailoutPrologue_);
// Emit debug trap handler code (target of patchable call instructions). This
// is just a tail call to the debug trap handler trampoline code.
{
JitRuntime* jrt = cx->runtime()->jitRuntime();
JitCode* handlerCode =
jrt->debugTrapHandler(cx, DebugTrapHandlerKind::Interpreter);
if (!handlerCode) {
return false;
}
debugTrapHandlerOffset_ = masm.currentOffset();
masm.jump(handlerCode);
}
// Emit the table.
masm.haltingAlign(sizeof(void*));
#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64)
size_t numInstructions = JSOP_LIMIT * (sizeof(uintptr_t) / sizeof(uint32_t));
AutoForbidPoolsAndNops afp(&masm, numInstructions);
#endif
tableOffset_ = masm.currentOffset();
for (size_t i = 0; i < JSOP_LIMIT; i++) {
const Label& opLabel = opLabels[i];
MOZ_ASSERT(opLabel.bound());
CodeLabel cl;
masm.writeCodePointer(&cl);
cl.target()->bind(opLabel.offset());
masm.addCodeLabel(cl);
}
return true;
}
void BaselineInterpreterGenerator::emitOutOfLineCodeCoverageInstrumentation() {
AutoCreatedBy acb(masm,
"BaselineInterpreterGenerator::"
"emitOutOfLineCodeCoverageInstrumentation");
masm.bind(handler.codeCoverageAtPrologueLabel());
#ifdef JS_USE_LINK_REGISTER
masm.pushReturnAddress();
#endif
saveInterpreterPCReg();
using Fn1 = void (*)(BaselineFrame* frame);
masm.setupUnalignedABICall(R0.scratchReg());
masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
masm.passABIArg(R0.scratchReg());
masm.callWithABI<Fn1, HandleCodeCoverageAtPrologue>();
restoreInterpreterPCReg();
masm.ret();
masm.bind(handler.codeCoverageAtPCLabel());
#ifdef JS_USE_LINK_REGISTER
masm.pushReturnAddress();
#endif
saveInterpreterPCReg();
using Fn2 = void (*)(BaselineFrame* frame, jsbytecode* pc);
masm.setupUnalignedABICall(R0.scratchReg());
masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
masm.passABIArg(R0.scratchReg());
Register pcReg = LoadBytecodePC(masm, R2.scratchReg());
masm.passABIArg(pcReg);
masm.callWithABI<Fn2, HandleCodeCoverageAtPC>();
restoreInterpreterPCReg();
masm.ret();
}
bool BaselineInterpreterGenerator::generate(BaselineInterpreter& interpreter) {
AutoCreatedBy acb(masm, "BaselineInterpreterGenerator::generate");
perfSpewer_.recordOffset(masm, "Prologue");
if (!emitPrologue()) {
return false;
}
perfSpewer_.recordOffset(masm, "InterpreterLoop");
if (!emitInterpreterLoop()) {
return false;
}
perfSpewer_.recordOffset(masm, "Epilogue");
if (!emitEpilogue()) {
return false;
}
perfSpewer_.recordOffset(masm, "OOLPostBarrierSlot");
if (!emitOutOfLinePostBarrierSlot()) {
return false;
}
perfSpewer_.recordOffset(masm, "OOLCodeCoverageInstrumentation");
emitOutOfLineCodeCoverageInstrumentation();
{
AutoCreatedBy acb(masm, "everything_else");
Linker linker(masm);
if (masm.oom()) {
ReportOutOfMemory(cx);
return false;
}
JitCode* code = linker.newCode(cx, CodeKind::Other);
if (!code) {
return false;
}
// Register BaselineInterpreter code with the profiler's JitCode table.
{
auto entry = MakeJitcodeGlobalEntry<BaselineInterpreterEntry>(
cx, code, code->raw(), code->rawEnd());
if (!entry) {
return false;
}
JitcodeGlobalTable* globalTable =
cx->runtime()->jitRuntime()->getJitcodeGlobalTable();
if (!globalTable->addEntry(std::move(entry))) {
ReportOutOfMemory(cx);
return false;
}
code->setHasBytecodeMap();
}
// Patch loads now that we know the tableswitch base address.
CodeLocationLabel tableLoc(code, CodeOffset(tableOffset_));
for (CodeOffset off : tableLabels_) {
MacroAssembler::patchNearAddressMove(CodeLocationLabel(code, off),
tableLoc);
}
perfSpewer_.saveProfile(code);
#ifdef MOZ_VTUNE
vtune::MarkStub(code, "BaselineInterpreter");
#endif
interpreter.init(
code, interpretOpOffset_, interpretOpNoDebugTrapOffset_,
bailoutPrologueOffset_.offset(),
profilerEnterFrameToggleOffset_.offset(),
profilerExitFrameToggleOffset_.offset(), debugTrapHandlerOffset_,
std::move(handler.debugInstrumentationOffsets()),
std::move(debugTrapOffsets_), std::move(handler.codeCoverageOffsets()),
std::move(handler.icReturnOffsets()), handler.callVMOffsets());
}
if (cx->runtime()->geckoProfiler().enabled()) {
interpreter.toggleProfilerInstrumentation(true);
}
if (coverage::IsLCovEnabled()) {
interpreter.toggleCodeCoverageInstrumentationUnchecked(true);
}
return true;
}
JitCode* JitRuntime::generateDebugTrapHandler(JSContext* cx,
DebugTrapHandlerKind kind) {
TempAllocator temp(&cx->tempLifoAlloc());
StackMacroAssembler masm(cx, temp);
AutoCreatedBy acb(masm, "JitRuntime::generateDebugTrapHandler");
AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
MOZ_ASSERT(!regs.has(FramePointer));
regs.takeUnchecked(ICStubReg);
if (HasInterpreterPCReg()) {
regs.takeUnchecked(InterpreterPCReg);
}
#ifdef JS_CODEGEN_ARM
regs.takeUnchecked(BaselineSecondScratchReg);
AutoNonDefaultSecondScratchRegister andssr(masm, BaselineSecondScratchReg);
#endif
Register scratch1 = regs.takeAny();
Register scratch2 = regs.takeAny();
Register scratch3 = regs.takeAny();
if (kind == DebugTrapHandlerKind::Interpreter) {
// The interpreter calls this for every script when debugging, so check if
// the script has any breakpoints or is in step mode before calling into
// C++.
Label hasDebugScript;
Address scriptAddr(FramePointer,
BaselineFrame::reverseOffsetOfInterpreterScript());
masm.loadPtr(scriptAddr, scratch1);
masm.branchTest32(Assembler::NonZero,
Address(scratch1, JSScript::offsetOfMutableFlags()),
Imm32(int32_t(JSScript::MutableFlags::HasDebugScript)),
&hasDebugScript);
masm.abiret();
masm.bind(&hasDebugScript);
if (HasInterpreterPCReg()) {
// Update frame's bytecode pc because the debugger depends on it.
Address pcAddr(FramePointer,
BaselineFrame::reverseOffsetOfInterpreterPC());
masm.storePtr(InterpreterPCReg, pcAddr);
}
}
// Load the return address in scratch1.
masm.loadAbiReturnAddress(scratch1);
// Load BaselineFrame pointer in scratch2.
masm.loadBaselineFramePtr(FramePointer, scratch2);
// Enter a stub frame and call the HandleDebugTrap VM function. Ensure
// the stub frame has a nullptr ICStub pointer, since this pointer is marked
// during GC.
masm.movePtr(ImmPtr(nullptr), ICStubReg);
EmitBaselineEnterStubFrame(masm, scratch3);
using Fn = bool (*)(JSContext*, BaselineFrame*, const uint8_t*);
VMFunctionId id = VMFunctionToId<Fn, jit::HandleDebugTrap>::id;
TrampolinePtr code = cx->runtime()->jitRuntime()->getVMWrapper(id);
masm.push(scratch1);
masm.push(scratch2);
EmitBaselineCallVM(code, masm);
EmitBaselineLeaveStubFrame(masm);
if (kind == DebugTrapHandlerKind::Interpreter) {
// We have to reload the bytecode pc register.
Address pcAddr(FramePointer, BaselineFrame::reverseOffsetOfInterpreterPC());
masm.loadPtr(pcAddr, InterpreterPCRegAtDispatch);
}
masm.abiret();
Linker linker(masm);
JitCode* handlerCode = linker.newCode(cx, CodeKind::Other);
if (!handlerCode) {
return nullptr;
}
CollectPerfSpewerJitCodeProfile(handlerCode, "DebugTrapHandler");
#ifdef MOZ_VTUNE
vtune::MarkStub(handlerCode, "DebugTrapHandler");
#endif
return handlerCode;
}
} // namespace jit
} // namespace js