Source code
Revision control
Copy as Markdown
Other Tools
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: set ts=8 sts=2 et sw=2 tw=80:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
#include "jit/IonCacheIRCompiler.h"
#include "mozilla/Maybe.h"
#include <algorithm>
#include "jit/CacheIRCompiler.h"
#include "jit/CacheIRWriter.h"
#include "jit/IonIC.h"
#include "jit/JitcodeMap.h"
#include "jit/JitFrames.h"
#include "jit/JitRuntime.h"
#include "jit/JitZone.h"
#include "jit/JSJitFrameIter.h"
#include "jit/Linker.h"
#include "jit/SharedICHelpers.h"
#include "jit/VMFunctions.h"
#include "proxy/DeadObjectProxy.h"
#include "proxy/Proxy.h"
#include "util/Memory.h"
#include "vm/StaticStrings.h"
#include "jit/JSJitFrameIter-inl.h"
#include "jit/MacroAssembler-inl.h"
#include "jit/VMFunctionList-inl.h"
using namespace js;
using namespace js::jit;
using mozilla::Maybe;
namespace JS {
struct ExpandoAndGeneration;
}
using JS::ExpandoAndGeneration;
namespace js {
namespace jit {
// IonCacheIRCompiler compiles CacheIR to IonIC native code.
IonCacheIRCompiler::IonCacheIRCompiler(JSContext* cx, TempAllocator& alloc,
const CacheIRWriter& writer, IonIC* ic,
IonScript* ionScript,
uint32_t stubDataOffset)
: CacheIRCompiler(cx, alloc, writer, stubDataOffset, Mode::Ion,
StubFieldPolicy::Constant),
writer_(writer),
ic_(ic),
ionScript_(ionScript),
savedLiveRegs_(false),
localTracingSlots_(0),
perfSpewer_(ic->pc()) {
MOZ_ASSERT(ic_);
MOZ_ASSERT(ionScript_);
}
template <typename T>
T IonCacheIRCompiler::rawPointerStubField(uint32_t offset) {
static_assert(sizeof(T) == sizeof(uintptr_t), "T must have pointer size");
return (T)readStubWord(offset, StubField::Type::RawPointer);
}
template <typename T>
T IonCacheIRCompiler::rawInt64StubField(uint32_t offset) {
static_assert(sizeof(T) == sizeof(int64_t), "T musthave int64 size");
return (T)readStubInt64(offset, StubField::Type::RawInt64);
}
template <typename Fn, Fn fn>
void IonCacheIRCompiler::callVM(MacroAssembler& masm) {
VMFunctionId id = VMFunctionToId<Fn, fn>::id;
callVMInternal(masm, id);
}
void IonCacheIRCompiler::pushStubCodePointer() {
stubJitCodeOffset_.emplace(masm.PushWithPatch(ImmPtr((void*)-1)));
}
// AutoSaveLiveRegisters must be used when we make a call that can GC. The
// constructor ensures all live registers are stored on the stack (where the GC
// expects them) and the destructor restores these registers.
AutoSaveLiveRegisters::AutoSaveLiveRegisters(IonCacheIRCompiler& compiler)
: compiler_(compiler) {
MOZ_ASSERT(compiler_.liveRegs_.isSome());
MOZ_ASSERT(compiler_.ic_);
compiler_.allocator.saveIonLiveRegisters(
compiler_.masm, compiler_.liveRegs_.ref(),
compiler_.ic_->scratchRegisterForEntryJump(), compiler_.ionScript_);
compiler_.savedLiveRegs_ = true;
}
AutoSaveLiveRegisters::~AutoSaveLiveRegisters() {
MOZ_ASSERT(compiler_.stubJitCodeOffset_.isSome(),
"Must have pushed JitCode* pointer");
compiler_.allocator.restoreIonLiveRegisters(compiler_.masm,
compiler_.liveRegs_.ref());
MOZ_ASSERT_IF(!compiler_.masm.oom(), compiler_.masm.framePushed() ==
compiler_.ionScript_->frameSize());
}
} // namespace jit
} // namespace js
void CacheRegisterAllocator::saveIonLiveRegisters(MacroAssembler& masm,
LiveRegisterSet liveRegs,
Register scratch,
IonScript* ionScript) {
// We have to push all registers in liveRegs on the stack. It's possible we
// stored other values in our live registers and stored operands on the
// stack (where our live registers should go), so this requires some careful
// work. Try to keep it simple by taking one small step at a time.
// Step 1. Discard any dead operands so we can reuse their registers.
freeDeadOperandLocations(masm);
// Step 2. Figure out the size of our live regs. This is consistent with
// the fact that we're using storeRegsInMask to generate the save code and
// PopRegsInMask to generate the restore code.
size_t sizeOfLiveRegsInBytes =
MacroAssembler::PushRegsInMaskSizeInBytes(liveRegs);
MOZ_ASSERT(sizeOfLiveRegsInBytes > 0);
// Step 3. Ensure all non-input operands are on the stack.
size_t numInputs = writer_.numInputOperands();
for (size_t i = numInputs; i < operandLocations_.length(); i++) {
OperandLocation& loc = operandLocations_[i];
if (loc.isInRegister()) {
spillOperandToStack(masm, &loc);
}
}
// Step 4. Restore the register state, but don't discard the stack as
// non-input operands are stored there.
restoreInputState(masm, /* shouldDiscardStack = */ false);
// We just restored the input state, so no input operands should be stored
// on the stack.
#ifdef DEBUG
for (size_t i = 0; i < numInputs; i++) {
const OperandLocation& loc = operandLocations_[i];
MOZ_ASSERT(!loc.isOnStack());
}
#endif
// Step 5. At this point our register state is correct. Stack values,
// however, may cover the space where we have to store the live registers.
// Move them out of the way.
bool hasOperandOnStack = false;
for (size_t i = numInputs; i < operandLocations_.length(); i++) {
OperandLocation& loc = operandLocations_[i];
if (!loc.isOnStack()) {
continue;
}
hasOperandOnStack = true;
size_t operandSize = loc.stackSizeInBytes();
size_t operandStackPushed = loc.stackPushed();
MOZ_ASSERT(operandSize > 0);
MOZ_ASSERT(stackPushed_ >= operandStackPushed);
MOZ_ASSERT(operandStackPushed >= operandSize);
// If this operand doesn't cover the live register space, there's
// nothing to do.
if (operandStackPushed - operandSize >= sizeOfLiveRegsInBytes) {
MOZ_ASSERT(stackPushed_ > sizeOfLiveRegsInBytes);
continue;
}
// Reserve stack space for the live registers if needed.
if (sizeOfLiveRegsInBytes > stackPushed_) {
size_t extraBytes = sizeOfLiveRegsInBytes - stackPushed_;
MOZ_ASSERT((extraBytes % sizeof(uintptr_t)) == 0);
masm.subFromStackPtr(Imm32(extraBytes));
stackPushed_ += extraBytes;
}
// Push the operand below the live register space.
if (loc.kind() == OperandLocation::PayloadStack) {
masm.push(
Address(masm.getStackPointer(), stackPushed_ - operandStackPushed));
stackPushed_ += operandSize;
loc.setPayloadStack(stackPushed_, loc.payloadType());
continue;
}
MOZ_ASSERT(loc.kind() == OperandLocation::ValueStack);
masm.pushValue(
Address(masm.getStackPointer(), stackPushed_ - operandStackPushed));
stackPushed_ += operandSize;
loc.setValueStack(stackPushed_);
}
// Step 6. If we have any operands on the stack, adjust their stackPushed
// values to not include sizeOfLiveRegsInBytes (this simplifies code down
// the line). Then push/store the live registers.
if (hasOperandOnStack) {
MOZ_ASSERT(stackPushed_ > sizeOfLiveRegsInBytes);
stackPushed_ -= sizeOfLiveRegsInBytes;
for (size_t i = numInputs; i < operandLocations_.length(); i++) {
OperandLocation& loc = operandLocations_[i];
if (loc.isOnStack()) {
loc.adjustStackPushed(-int32_t(sizeOfLiveRegsInBytes));
}
}
size_t stackBottom = stackPushed_ + sizeOfLiveRegsInBytes;
masm.storeRegsInMask(liveRegs, Address(masm.getStackPointer(), stackBottom),
scratch);
masm.setFramePushed(masm.framePushed() + sizeOfLiveRegsInBytes);
} else {
// If no operands are on the stack, discard the unused stack space.
if (stackPushed_ > 0) {
masm.addToStackPtr(Imm32(stackPushed_));
stackPushed_ = 0;
}
masm.PushRegsInMask(liveRegs);
}
freePayloadSlots_.clear();
freeValueSlots_.clear();
MOZ_ASSERT_IF(!masm.oom(), masm.framePushed() == ionScript->frameSize() +
sizeOfLiveRegsInBytes);
// Step 7. All live registers and non-input operands are stored on the stack
// now, so at this point all registers except for the input registers are
// available.
availableRegs_.set() = GeneralRegisterSet::Not(inputRegisterSet());
availableRegsAfterSpill_.set() = GeneralRegisterSet();
// Step 8. We restored our input state, so we have to fix up aliased input
// registers again.
fixupAliasedInputs(masm);
}
void CacheRegisterAllocator::restoreIonLiveRegisters(MacroAssembler& masm,
LiveRegisterSet liveRegs) {
masm.PopRegsInMask(liveRegs);
availableRegs_.set() = GeneralRegisterSet();
availableRegsAfterSpill_.set() = GeneralRegisterSet::All();
}
static void* GetReturnAddressToIonCode(JSContext* cx) {
JSJitFrameIter frame(cx->activation()->asJit());
MOZ_ASSERT(frame.type() == FrameType::Exit,
"An exit frame is expected as update functions are called with a "
"VMFunction.");
void* returnAddr = frame.returnAddress();
#ifdef DEBUG
++frame;
MOZ_ASSERT(frame.isIonJS());
#endif
return returnAddr;
}
// The AutoSaveLiveRegisters parameter is used to ensure registers were saved
void IonCacheIRCompiler::enterStubFrame(MacroAssembler& masm,
const AutoSaveLiveRegisters&) {
MOZ_ASSERT(!enteredStubFrame_);
pushStubCodePointer();
masm.PushFrameDescriptor(FrameType::IonJS);
masm.Push(ImmPtr(GetReturnAddressToIonCode(cx_)));
masm.Push(FramePointer);
masm.moveStackPtrTo(FramePointer);
enteredStubFrame_ = true;
}
void IonCacheIRCompiler::storeTracedValue(MacroAssembler& masm,
ValueOperand value) {
MOZ_ASSERT(localTracingSlots_ < 255);
masm.Push(value);
localTracingSlots_++;
}
void IonCacheIRCompiler::loadTracedValue(MacroAssembler& masm,
uint8_t slotIndex,
ValueOperand value) {
MOZ_ASSERT(slotIndex <= localTracingSlots_);
int32_t offset = IonICCallFrameLayout::LocallyTracedValueOffset +
slotIndex * sizeof(Value);
masm.loadValue(Address(FramePointer, -offset), value);
}
bool IonCacheIRCompiler::init() {
if (!allocator.init()) {
return false;
}
size_t numInputs = writer_.numInputOperands();
MOZ_ASSERT(numInputs == NumInputsForCacheKind(ic_->kind()));
AllocatableGeneralRegisterSet available;
switch (ic_->kind()) {
case CacheKind::GetProp:
case CacheKind::GetElem: {
IonGetPropertyIC* ic = ic_->asGetPropertyIC();
ValueOperand output = ic->output();
available.add(output);
liveRegs_.emplace(ic->liveRegs());
outputUnchecked_.emplace(output);
MOZ_ASSERT(numInputs == 1 || numInputs == 2);
allocator.initInputLocation(0, ic->value());
if (numInputs > 1) {
allocator.initInputLocation(1, ic->id());
}
break;
}
case CacheKind::GetPropSuper:
case CacheKind::GetElemSuper: {
IonGetPropSuperIC* ic = ic_->asGetPropSuperIC();
ValueOperand output = ic->output();
available.add(output);
liveRegs_.emplace(ic->liveRegs());
outputUnchecked_.emplace(output);
MOZ_ASSERT(numInputs == 2 || numInputs == 3);
allocator.initInputLocation(0, ic->object(), JSVAL_TYPE_OBJECT);
if (ic->kind() == CacheKind::GetPropSuper) {
MOZ_ASSERT(numInputs == 2);
allocator.initInputLocation(1, ic->receiver());
} else {
MOZ_ASSERT(numInputs == 3);
allocator.initInputLocation(1, ic->id());
allocator.initInputLocation(2, ic->receiver());
}
break;
}
case CacheKind::SetProp:
case CacheKind::SetElem: {
IonSetPropertyIC* ic = ic_->asSetPropertyIC();
available.add(ic->temp());
liveRegs_.emplace(ic->liveRegs());
allocator.initInputLocation(0, ic->object(), JSVAL_TYPE_OBJECT);
if (ic->kind() == CacheKind::SetProp) {
MOZ_ASSERT(numInputs == 2);
allocator.initInputLocation(1, ic->rhs());
} else {
MOZ_ASSERT(numInputs == 3);
allocator.initInputLocation(1, ic->id());
allocator.initInputLocation(2, ic->rhs());
}
break;
}
case CacheKind::GetName: {
IonGetNameIC* ic = ic_->asGetNameIC();
ValueOperand output = ic->output();
available.add(output);
available.add(ic->temp());
liveRegs_.emplace(ic->liveRegs());
outputUnchecked_.emplace(output);
MOZ_ASSERT(numInputs == 1);
allocator.initInputLocation(0, ic->environment(), JSVAL_TYPE_OBJECT);
break;
}
case CacheKind::BindName: {
IonBindNameIC* ic = ic_->asBindNameIC();
Register output = ic->output();
available.add(output);
available.add(ic->temp());
liveRegs_.emplace(ic->liveRegs());
outputUnchecked_.emplace(
TypedOrValueRegister(MIRType::Object, AnyRegister(output)));
MOZ_ASSERT(numInputs == 1);
allocator.initInputLocation(0, ic->environment(), JSVAL_TYPE_OBJECT);
break;
}
case CacheKind::GetIterator: {
IonGetIteratorIC* ic = ic_->asGetIteratorIC();
Register output = ic->output();
available.add(output);
available.add(ic->temp1());
available.add(ic->temp2());
liveRegs_.emplace(ic->liveRegs());
outputUnchecked_.emplace(
TypedOrValueRegister(MIRType::Object, AnyRegister(output)));
MOZ_ASSERT(numInputs == 1);
allocator.initInputLocation(0, ic->value());
break;
}
case CacheKind::OptimizeSpreadCall: {
auto* ic = ic_->asOptimizeSpreadCallIC();
ValueOperand output = ic->output();
available.add(output);
available.add(ic->temp());
liveRegs_.emplace(ic->liveRegs());
outputUnchecked_.emplace(output);
MOZ_ASSERT(numInputs == 1);
allocator.initInputLocation(0, ic->value());
break;
}
case CacheKind::In: {
IonInIC* ic = ic_->asInIC();
Register output = ic->output();
available.add(output);
liveRegs_.emplace(ic->liveRegs());
outputUnchecked_.emplace(
TypedOrValueRegister(MIRType::Boolean, AnyRegister(output)));
MOZ_ASSERT(numInputs == 2);
allocator.initInputLocation(0, ic->key());
allocator.initInputLocation(
1, TypedOrValueRegister(MIRType::Object, AnyRegister(ic->object())));
break;
}
case CacheKind::HasOwn: {
IonHasOwnIC* ic = ic_->asHasOwnIC();
Register output = ic->output();
available.add(output);
liveRegs_.emplace(ic->liveRegs());
outputUnchecked_.emplace(
TypedOrValueRegister(MIRType::Boolean, AnyRegister(output)));
MOZ_ASSERT(numInputs == 2);
allocator.initInputLocation(0, ic->id());
allocator.initInputLocation(1, ic->value());
break;
}
case CacheKind::CheckPrivateField: {
IonCheckPrivateFieldIC* ic = ic_->asCheckPrivateFieldIC();
Register output = ic->output();
available.add(output);
liveRegs_.emplace(ic->liveRegs());
outputUnchecked_.emplace(
TypedOrValueRegister(MIRType::Boolean, AnyRegister(output)));
MOZ_ASSERT(numInputs == 2);
allocator.initInputLocation(0, ic->value());
allocator.initInputLocation(1, ic->id());
break;
}
case CacheKind::InstanceOf: {
IonInstanceOfIC* ic = ic_->asInstanceOfIC();
Register output = ic->output();
available.add(output);
liveRegs_.emplace(ic->liveRegs());
outputUnchecked_.emplace(
TypedOrValueRegister(MIRType::Boolean, AnyRegister(output)));
MOZ_ASSERT(numInputs == 2);
allocator.initInputLocation(0, ic->lhs());
allocator.initInputLocation(
1, TypedOrValueRegister(MIRType::Object, AnyRegister(ic->rhs())));
break;
}
case CacheKind::ToPropertyKey: {
IonToPropertyKeyIC* ic = ic_->asToPropertyKeyIC();
ValueOperand output = ic->output();
available.add(output);
liveRegs_.emplace(ic->liveRegs());
outputUnchecked_.emplace(TypedOrValueRegister(output));
MOZ_ASSERT(numInputs == 1);
allocator.initInputLocation(0, ic->input());
break;
}
case CacheKind::UnaryArith: {
IonUnaryArithIC* ic = ic_->asUnaryArithIC();
ValueOperand output = ic->output();
available.add(output);
liveRegs_.emplace(ic->liveRegs());
outputUnchecked_.emplace(TypedOrValueRegister(output));
MOZ_ASSERT(numInputs == 1);
allocator.initInputLocation(0, ic->input());
break;
}
case CacheKind::BinaryArith: {
IonBinaryArithIC* ic = ic_->asBinaryArithIC();
ValueOperand output = ic->output();
available.add(output);
liveRegs_.emplace(ic->liveRegs());
outputUnchecked_.emplace(TypedOrValueRegister(output));
MOZ_ASSERT(numInputs == 2);
allocator.initInputLocation(0, ic->lhs());
allocator.initInputLocation(1, ic->rhs());
break;
}
case CacheKind::Compare: {
IonCompareIC* ic = ic_->asCompareIC();
Register output = ic->output();
available.add(output);
liveRegs_.emplace(ic->liveRegs());
outputUnchecked_.emplace(
TypedOrValueRegister(MIRType::Boolean, AnyRegister(output)));
MOZ_ASSERT(numInputs == 2);
allocator.initInputLocation(0, ic->lhs());
allocator.initInputLocation(1, ic->rhs());
break;
}
case CacheKind::CloseIter: {
IonCloseIterIC* ic = ic_->asCloseIterIC();
available.add(ic->temp());
liveRegs_.emplace(ic->liveRegs());
allocator.initInputLocation(0, ic->iter(), JSVAL_TYPE_OBJECT);
break;
}
case CacheKind::OptimizeGetIterator: {
auto* ic = ic_->asOptimizeGetIteratorIC();
Register output = ic->output();
available.add(output);
available.add(ic->temp());
liveRegs_.emplace(ic->liveRegs());
outputUnchecked_.emplace(
TypedOrValueRegister(MIRType::Boolean, AnyRegister(output)));
MOZ_ASSERT(numInputs == 1);
allocator.initInputLocation(0, ic->value());
break;
}
case CacheKind::Call:
case CacheKind::TypeOf:
case CacheKind::TypeOfEq:
case CacheKind::ToBool:
case CacheKind::GetIntrinsic:
case CacheKind::NewArray:
case CacheKind::NewObject:
MOZ_CRASH("Unsupported IC");
}
liveFloatRegs_ = LiveFloatRegisterSet(liveRegs_->fpus());
allocator.initAvailableRegs(available);
allocator.initAvailableRegsAfterSpill();
return true;
}
JitCode* IonCacheIRCompiler::compile(IonICStub* stub) {
AutoCreatedBy acb(masm, "IonCacheIRCompiler::compile");
masm.setFramePushed(ionScript_->frameSize());
if (cx_->runtime()->geckoProfiler().enabled()) {
masm.enableProfilingInstrumentation();
}
allocator.fixupAliasedInputs(masm);
CacheIRReader reader(writer_);
do {
CacheOp op = reader.readOp();
perfSpewer_.recordInstruction(masm, op);
switch (op) {
#define DEFINE_OP(op, ...) \
case CacheOp::op: \
if (!emit##op(reader)) return nullptr; \
break;