Source code
Revision control
Copy as Markdown
Other Tools
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: set ts=8 sts=2 et sw=2 tw=80:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
#include "jit/arm/CodeGenerator-arm.h"
#include "mozilla/DebugOnly.h"
#include "mozilla/MathAlgorithms.h"
#include "mozilla/Maybe.h"
#include <iterator>
#include "jsnum.h"
#include "jit/CodeGenerator.h"
#include "jit/InlineScriptTree.h"
#include "jit/JitRuntime.h"
#include "jit/MIR.h"
#include "jit/MIRGraph.h"
#include "js/Conversions.h"
#include "js/ScalarType.h" // js::Scalar::Type
#include "vm/JSContext.h"
#include "vm/Realm.h"
#include "vm/Shape.h"
#include "jit/MacroAssembler-inl.h"
#include "jit/shared/CodeGenerator-shared-inl.h"
#include "vm/JSScript-inl.h"
using namespace js;
using namespace js::jit;
using JS::GenericNaN;
using JS::ToInt32;
using mozilla::DebugOnly;
using mozilla::FloorLog2;
using mozilla::NegativeInfinity;
// shared
CodeGeneratorARM::CodeGeneratorARM(MIRGenerator* gen, LIRGraph* graph,
MacroAssembler* masm)
: CodeGeneratorShared(gen, graph, masm) {}
Register64 CodeGeneratorARM::ToOperandOrRegister64(
const LInt64Allocation input) {
return ToRegister64(input);
}
void CodeGeneratorARM::emitBranch(Assembler::Condition cond,
MBasicBlock* mirTrue, MBasicBlock* mirFalse) {
if (isNextBlock(mirFalse->lir())) {
jumpToBlock(mirTrue, cond);
} else {
jumpToBlock(mirFalse, Assembler::InvertCondition(cond));
jumpToBlock(mirTrue);
}
}
void OutOfLineBailout::accept(CodeGeneratorARM* codegen) {
codegen->visitOutOfLineBailout(this);
}
void CodeGenerator::visitTestIAndBranch(LTestIAndBranch* test) {
const LAllocation* opd = test->getOperand(0);
MBasicBlock* ifTrue = test->ifTrue();
MBasicBlock* ifFalse = test->ifFalse();
// Test the operand
masm.as_cmp(ToRegister(opd), Imm8(0));
if (isNextBlock(ifFalse->lir())) {
jumpToBlock(ifTrue, Assembler::NonZero);
} else if (isNextBlock(ifTrue->lir())) {
jumpToBlock(ifFalse, Assembler::Zero);
} else {
jumpToBlock(ifFalse, Assembler::Zero);
jumpToBlock(ifTrue);
}
}
void CodeGenerator::visitCompare(LCompare* comp) {
Assembler::Condition cond =
JSOpToCondition(comp->mir()->compareType(), comp->jsop());
const LAllocation* left = comp->getOperand(0);
const LAllocation* right = comp->getOperand(1);
const LDefinition* def = comp->getDef(0);
ScratchRegisterScope scratch(masm);
if (right->isConstant()) {
masm.ma_cmp(ToRegister(left), Imm32(ToInt32(right)), scratch);
} else if (right->isRegister()) {
masm.ma_cmp(ToRegister(left), ToRegister(right));
} else {
SecondScratchRegisterScope scratch2(masm);
masm.ma_cmp(ToRegister(left), Operand(ToAddress(right)), scratch, scratch2);
}
masm.ma_mov(Imm32(0), ToRegister(def));
masm.ma_mov(Imm32(1), ToRegister(def), cond);
}
void CodeGenerator::visitCompareAndBranch(LCompareAndBranch* comp) {
Assembler::Condition cond =
JSOpToCondition(comp->cmpMir()->compareType(), comp->jsop());
const LAllocation* left = comp->left();
const LAllocation* right = comp->right();
ScratchRegisterScope scratch(masm);
if (right->isConstant()) {
masm.ma_cmp(ToRegister(left), Imm32(ToInt32(right)), scratch);
} else if (right->isRegister()) {
masm.ma_cmp(ToRegister(left), ToRegister(right));
} else {
SecondScratchRegisterScope scratch2(masm);
masm.ma_cmp(ToRegister(left), Operand(ToAddress(right)), scratch, scratch2);
}
emitBranch(cond, comp->ifTrue(), comp->ifFalse());
}
bool CodeGeneratorARM::generateOutOfLineCode() {
if (!CodeGeneratorShared::generateOutOfLineCode()) {
return false;
}
if (deoptLabel_.used()) {
// All non-table-based bailouts will go here.
masm.bind(&deoptLabel_);
// Push the frame size, so the handler can recover the IonScript.
masm.push(Imm32(frameSize()));
TrampolinePtr handler = gen->jitRuntime()->getGenericBailoutHandler();
masm.jump(handler);
}
return !masm.oom();
}
void CodeGeneratorARM::bailoutIf(Assembler::Condition condition,
LSnapshot* snapshot) {
encode(snapshot);
InlineScriptTree* tree = snapshot->mir()->block()->trackedTree();
OutOfLineBailout* ool =
new (alloc()) OutOfLineBailout(snapshot, masm.framePushed());
// All bailout code is associated with the bytecodeSite of the block we are
// bailing out from.
addOutOfLineCode(ool,
new (alloc()) BytecodeSite(tree, tree->script()->code()));
masm.ma_b(ool->entry(), condition);
}
void CodeGeneratorARM::bailoutFrom(Label* label, LSnapshot* snapshot) {
MOZ_ASSERT_IF(!masm.oom(), label->used());
MOZ_ASSERT_IF(!masm.oom(), !label->bound());
encode(snapshot);
InlineScriptTree* tree = snapshot->mir()->block()->trackedTree();
OutOfLineBailout* ool =
new (alloc()) OutOfLineBailout(snapshot, masm.framePushed());
// All bailout code is associated with the bytecodeSite of the block we are
// bailing out from.
addOutOfLineCode(ool,
new (alloc()) BytecodeSite(tree, tree->script()->code()));
masm.retarget(label, ool->entry());
}
void CodeGeneratorARM::bailout(LSnapshot* snapshot) {
Label label;
masm.ma_b(&label);
bailoutFrom(&label, snapshot);
}
void CodeGeneratorARM::visitOutOfLineBailout(OutOfLineBailout* ool) {
masm.push(Imm32(ool->snapshot()->snapshotOffset()));
masm.ma_b(&deoptLabel_);
}
void CodeGenerator::visitMinMaxD(LMinMaxD* ins) {
FloatRegister first = ToFloatRegister(ins->first());
FloatRegister second = ToFloatRegister(ins->second());
MOZ_ASSERT(first == ToFloatRegister(ins->output()));
if (ins->mir()->isMax()) {
masm.maxDouble(second, first, true);
} else {
masm.minDouble(second, first, true);
}
}
void CodeGenerator::visitMinMaxF(LMinMaxF* ins) {
FloatRegister first = ToFloatRegister(ins->first());
FloatRegister second = ToFloatRegister(ins->second());
MOZ_ASSERT(first == ToFloatRegister(ins->output()));
if (ins->mir()->isMax()) {
masm.maxFloat32(second, first, true);
} else {
masm.minFloat32(second, first, true);
}
}
void CodeGenerator::visitAddI(LAddI* ins) {
const LAllocation* lhs = ins->getOperand(0);
const LAllocation* rhs = ins->getOperand(1);
const LDefinition* dest = ins->getDef(0);
ScratchRegisterScope scratch(masm);
if (rhs->isConstant()) {
masm.ma_add(ToRegister(lhs), Imm32(ToInt32(rhs)), ToRegister(dest), scratch,
SetCC);
} else if (rhs->isRegister()) {
masm.ma_add(ToRegister(lhs), ToRegister(rhs), ToRegister(dest), SetCC);
} else {
masm.ma_add(ToRegister(lhs), Operand(ToAddress(rhs)), ToRegister(dest),
SetCC);
}
if (ins->snapshot()) {
bailoutIf(Assembler::Overflow, ins->snapshot());
}
}
void CodeGenerator::visitAddI64(LAddI64* lir) {
const LInt64Allocation lhs = lir->getInt64Operand(LAddI64::Lhs);
const LInt64Allocation rhs = lir->getInt64Operand(LAddI64::Rhs);
MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
if (IsConstant(rhs)) {
masm.add64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
return;
}
masm.add64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
}
void CodeGenerator::visitSubI(LSubI* ins) {
const LAllocation* lhs = ins->getOperand(0);
const LAllocation* rhs = ins->getOperand(1);
const LDefinition* dest = ins->getDef(0);
ScratchRegisterScope scratch(masm);
if (rhs->isConstant()) {
masm.ma_sub(ToRegister(lhs), Imm32(ToInt32(rhs)), ToRegister(dest), scratch,
SetCC);
} else if (rhs->isRegister()) {
masm.ma_sub(ToRegister(lhs), ToRegister(rhs), ToRegister(dest), SetCC);
} else {
masm.ma_sub(ToRegister(lhs), Operand(ToAddress(rhs)), ToRegister(dest),
SetCC);
}
if (ins->snapshot()) {
bailoutIf(Assembler::Overflow, ins->snapshot());
}
}
void CodeGenerator::visitSubI64(LSubI64* lir) {
const LInt64Allocation lhs = lir->getInt64Operand(LSubI64::Lhs);
const LInt64Allocation rhs = lir->getInt64Operand(LSubI64::Rhs);
MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
if (IsConstant(rhs)) {
masm.sub64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
return;
}
masm.sub64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
}
void CodeGenerator::visitMulI(LMulI* ins) {
const LAllocation* lhs = ins->getOperand(0);
const LAllocation* rhs = ins->getOperand(1);
const LDefinition* dest = ins->getDef(0);
MMul* mul = ins->mir();
MOZ_ASSERT_IF(mul->mode() == MMul::Integer,
!mul->canBeNegativeZero() && !mul->canOverflow());
if (rhs->isConstant()) {
// Bailout when this condition is met.
Assembler::Condition c = Assembler::Overflow;
// Bailout on -0.0
int32_t constant = ToInt32(rhs);
if (mul->canBeNegativeZero() && constant <= 0) {
Assembler::Condition bailoutCond =
(constant == 0) ? Assembler::LessThan : Assembler::Equal;
masm.as_cmp(ToRegister(lhs), Imm8(0));
bailoutIf(bailoutCond, ins->snapshot());
}
// TODO: move these to ma_mul.
switch (constant) {
case -1:
masm.as_rsb(ToRegister(dest), ToRegister(lhs), Imm8(0), SetCC);
break;
case 0:
masm.ma_mov(Imm32(0), ToRegister(dest));
return; // Escape overflow check;
case 1:
// Nop
masm.ma_mov(ToRegister(lhs), ToRegister(dest));
return; // Escape overflow check;
case 2:
masm.ma_add(ToRegister(lhs), ToRegister(lhs), ToRegister(dest), SetCC);
// Overflow is handled later.
break;
default: {
bool handled = false;
if (constant > 0) {
// Try shift and add sequences for a positive constant.
if (!mul->canOverflow()) {
// If it cannot overflow, we can do lots of optimizations.
Register src = ToRegister(lhs);
uint32_t shift = FloorLog2(constant);
uint32_t rest = constant - (1 << shift);
// See if the constant has one bit set, meaning it can be
// encoded as a bitshift.
if ((1 << shift) == constant) {
masm.ma_lsl(Imm32(shift), src, ToRegister(dest));
handled = true;
} else {
// If the constant cannot be encoded as (1 << C1), see
// if it can be encoded as (1 << C1) | (1 << C2), which
// can be computed using an add and a shift.
uint32_t shift_rest = FloorLog2(rest);
if ((1u << shift_rest) == rest) {
masm.as_add(ToRegister(dest), src,
lsl(src, shift - shift_rest));
if (shift_rest != 0) {
masm.ma_lsl(Imm32(shift_rest), ToRegister(dest),
ToRegister(dest));
}
handled = true;
}
}
} else if (ToRegister(lhs) != ToRegister(dest)) {
// To stay on the safe side, only optimize things that are a
// power of 2.
uint32_t shift = FloorLog2(constant);
if ((1 << shift) == constant) {
// dest = lhs * pow(2,shift)
masm.ma_lsl(Imm32(shift), ToRegister(lhs), ToRegister(dest));
// At runtime, check (lhs == dest >> shift), if this
// does not hold, some bits were lost due to overflow,
// and the computation should be resumed as a double.
masm.as_cmp(ToRegister(lhs), asr(ToRegister(dest), shift));
c = Assembler::NotEqual;
handled = true;
}
}
}
if (!handled) {
ScratchRegisterScope scratch(masm);
if (mul->canOverflow()) {
c = masm.ma_check_mul(ToRegister(lhs), Imm32(ToInt32(rhs)),
ToRegister(dest), scratch, c);
} else {
masm.ma_mul(ToRegister(lhs), Imm32(ToInt32(rhs)), ToRegister(dest),
scratch);
}
}
}
}
// Bailout on overflow.
if (mul->canOverflow()) {
bailoutIf(c, ins->snapshot());
}
} else {
Assembler::Condition c = Assembler::Overflow;
if (mul->canOverflow()) {
ScratchRegisterScope scratch(masm);
c = masm.ma_check_mul(ToRegister(lhs), ToRegister(rhs), ToRegister(dest),
scratch, c);
} else {
masm.ma_mul(ToRegister(lhs), ToRegister(rhs), ToRegister(dest));
}
// Bailout on overflow.
if (mul->canOverflow()) {
bailoutIf(c, ins->snapshot());
}
if (mul->canBeNegativeZero()) {
Label done;
masm.as_cmp(ToRegister(dest), Imm8(0));
masm.ma_b(&done, Assembler::NotEqual);
// Result is -0 if lhs or rhs is negative.
masm.ma_cmn(ToRegister(lhs), ToRegister(rhs));
bailoutIf(Assembler::Signed, ins->snapshot());
masm.bind(&done);
}
}
}
void CodeGenerator::visitMulI64(LMulI64* lir) {
const LInt64Allocation lhs = lir->getInt64Operand(LMulI64::Lhs);
const LInt64Allocation rhs = lir->getInt64Operand(LMulI64::Rhs);
MOZ_ASSERT(ToRegister64(lhs) == ToOutRegister64(lir));
if (IsConstant(rhs)) {
int64_t constant = ToInt64(rhs);
switch (constant) {
case -1:
masm.neg64(ToRegister64(lhs));
return;
case 0:
masm.xor64(ToRegister64(lhs), ToRegister64(lhs));
return;
case 1:
// nop
return;
case 2:
masm.add64(ToRegister64(lhs), ToRegister64(lhs));
return;
default:
if (constant > 0) {
// Use shift if constant is power of 2.
int32_t shift = mozilla::FloorLog2(constant);
if (int64_t(1) << shift == constant) {
masm.lshift64(Imm32(shift), ToRegister64(lhs));
return;
}
}
Register temp = ToTempRegisterOrInvalid(lir->temp());
masm.mul64(Imm64(constant), ToRegister64(lhs), temp);
}
} else {
Register temp = ToTempRegisterOrInvalid(lir->temp());
masm.mul64(ToOperandOrRegister64(rhs), ToRegister64(lhs), temp);
}
}
void CodeGeneratorARM::divICommon(MDiv* mir, Register lhs, Register rhs,
Register output, LSnapshot* snapshot,
Label& done) {
ScratchRegisterScope scratch(masm);
if (mir->canBeNegativeOverflow()) {
// Handle INT32_MIN / -1;
// The integer division will give INT32_MIN, but we want -(double)INT32_MIN.
// Sets EQ if lhs == INT32_MIN.
masm.ma_cmp(lhs, Imm32(INT32_MIN), scratch);
// If EQ (LHS == INT32_MIN), sets EQ if rhs == -1.
masm.ma_cmp(rhs, Imm32(-1), scratch, Assembler::Equal);
if (mir->canTruncateOverflow()) {
if (mir->trapOnError()) {
Label ok;
masm.ma_b(&ok, Assembler::NotEqual);
masm.wasmTrap(wasm::Trap::IntegerOverflow, mir->bytecodeOffset());
masm.bind(&ok);
} else {
// (-INT32_MIN)|0 = INT32_MIN
Label skip;
masm.ma_b(&skip, Assembler::NotEqual);
masm.ma_mov(Imm32(INT32_MIN), output);
masm.ma_b(&done);
masm.bind(&skip);
}
} else {
MOZ_ASSERT(mir->fallible());
bailoutIf(Assembler::Equal, snapshot);
}
}
// Handle divide by zero.
if (mir->canBeDivideByZero()) {
masm.as_cmp(rhs, Imm8(0));
if (mir->canTruncateInfinities()) {
if (mir->trapOnError()) {
Label nonZero;
masm.ma_b(&nonZero, Assembler::NotEqual);
masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->bytecodeOffset());
masm.bind(&nonZero);
} else {
// Infinity|0 == 0
Label skip;
masm.ma_b(&skip, Assembler::NotEqual);
masm.ma_mov(Imm32(0), output);
masm.ma_b(&done);
masm.bind(&skip);
}
} else {
MOZ_ASSERT(mir->fallible());
bailoutIf(Assembler::Equal, snapshot);
}
}
// Handle negative 0.
if (!mir->canTruncateNegativeZero() && mir->canBeNegativeZero()) {
Label nonzero;
masm.as_cmp(lhs, Imm8(0));
masm.ma_b(&nonzero, Assembler::NotEqual);
masm.as_cmp(rhs, Imm8(0));
MOZ_ASSERT(mir->fallible());
bailoutIf(Assembler::LessThan, snapshot);
masm.bind(&nonzero);
}
}
void CodeGenerator::visitDivI(LDivI* ins) {
Register lhs = ToRegister(ins->lhs());
Register rhs = ToRegister(ins->rhs());
Register temp = ToRegister(ins->getTemp(0));
Register output = ToRegister(ins->output());
MDiv* mir = ins->mir();
Label done;
divICommon(mir, lhs, rhs, output, ins->snapshot(), done);
if (mir->canTruncateRemainder()) {
masm.ma_sdiv(lhs, rhs, output);
} else {
{
ScratchRegisterScope scratch(masm);
masm.ma_sdiv(lhs, rhs, temp);
masm.ma_mul(temp, rhs, scratch);
masm.ma_cmp(lhs, scratch);
}
bailoutIf(Assembler::NotEqual, ins->snapshot());
masm.ma_mov(temp, output);
}
masm.bind(&done);
}
extern "C" {
extern MOZ_EXPORT int64_t __aeabi_idivmod(int, int);
extern MOZ_EXPORT int64_t __aeabi_uidivmod(int, int);
}
void CodeGenerator::visitSoftDivI(LSoftDivI* ins) {
Register lhs = ToRegister(ins->lhs());
Register rhs = ToRegister(ins->rhs());
Register output = ToRegister(ins->output());
MDiv* mir = ins->mir();
Label done;
divICommon(mir, lhs, rhs, output, ins->snapshot(), done);
if (gen->compilingWasm()) {
masm.Push(InstanceReg);
int32_t framePushedAfterInstance = masm.framePushed();
masm.setupWasmABICall();
masm.passABIArg(lhs);
masm.passABIArg(rhs);
int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance;
masm.callWithABI(mir->bytecodeOffset(),
wasm::SymbolicAddress::aeabi_idivmod,
mozilla::Some(instanceOffset));
masm.Pop(InstanceReg);
} else {
using Fn = int64_t (*)(int, int);
masm.setupAlignedABICall();
masm.passABIArg(lhs);
masm.passABIArg(rhs);
masm.callWithABI<Fn, __aeabi_idivmod>(
MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckOther);
}
// idivmod returns the quotient in r0, and the remainder in r1.
if (!mir->canTruncateRemainder()) {
MOZ_ASSERT(mir->fallible());
masm.as_cmp(r1, Imm8(0));
bailoutIf(Assembler::NonZero, ins->snapshot());
}
masm.bind(&done);
}
void CodeGenerator::visitDivPowTwoI(LDivPowTwoI* ins) {
MDiv* mir = ins->mir();
Register lhs = ToRegister(ins->numerator());
Register output = ToRegister(ins->output());
int32_t shift = ins->shift();
if (shift == 0) {
masm.ma_mov(lhs, output);
return;
}
if (!mir->isTruncated()) {
// If the remainder is != 0, bailout since this must be a double.
{
// The bailout code also needs the scratch register.
// Here it is only used as a dummy target to set CC flags.
ScratchRegisterScope scratch(masm);
masm.as_mov(scratch, lsl(lhs, 32 - shift), SetCC);
}
bailoutIf(Assembler::NonZero, ins->snapshot());
}
if (!mir->canBeNegativeDividend()) {
// Numerator is unsigned, so needs no adjusting. Do the shift.
masm.as_mov(output, asr(lhs, shift));
return;
}
// Adjust the value so that shifting produces a correctly rounded result
// when the numerator is negative. See 10-1 "Signed Division by a Known
// Power of 2" in Henry S. Warren, Jr.'s Hacker's Delight.
ScratchRegisterScope scratch(masm);
if (shift > 1) {
masm.as_mov(scratch, asr(lhs, 31));
masm.as_add(scratch, lhs, lsr(scratch, 32 - shift));
} else {
masm.as_add(scratch, lhs, lsr(lhs, 32 - shift));
}
// Do the shift.
masm.as_mov(output, asr(scratch, shift));
}
void CodeGeneratorARM::modICommon(MMod* mir, Register lhs, Register rhs,
Register output, LSnapshot* snapshot,
Label& done) {
// X % 0 is bad because it will give garbage (or abort), when it should give
// NaN.
if (mir->canBeDivideByZero()) {
masm.as_cmp(rhs, Imm8(0));
if (mir->isTruncated()) {
Label nonZero;
masm.ma_b(&nonZero, Assembler::NotEqual);
if (mir->trapOnError()) {
masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->bytecodeOffset());
} else {
// NaN|0 == 0
masm.ma_mov(Imm32(0), output);
masm.ma_b(&done);
}
masm.bind(&nonZero);
} else {
MOZ_ASSERT(mir->fallible());
bailoutIf(Assembler::Equal, snapshot);
}
}
}
void CodeGenerator::visitModI(LModI* ins) {
Register lhs = ToRegister(ins->lhs());
Register rhs = ToRegister(ins->rhs());
Register output = ToRegister(ins->output());
MMod* mir = ins->mir();
// Contrary to other architectures (notably x86) INT_MIN % -1 doesn't need to
// be handled separately. |ma_smod| computes the remainder using the |SDIV|
// and the |MLS| instructions. On overflow, |SDIV| truncates the result to
// 32-bit and returns INT_MIN, see ARM Architecture Reference Manual, SDIV
// instruction.
//
// mls(INT_MIN, sdiv(INT_MIN, -1), -1)
// = INT_MIN - (sdiv(INT_MIN, -1) * -1)
// = INT_MIN - (INT_MIN * -1)
// = INT_MIN - INT_MIN
// = 0
//
// And a zero remainder with a negative dividend is already handled below.
Label done;
modICommon(mir, lhs, rhs, output, ins->snapshot(), done);
{
ScratchRegisterScope scratch(masm);
masm.ma_smod(lhs, rhs, output, scratch);
}
// If X%Y == 0 and X < 0, then we *actually* wanted to return -0.0.
if (mir->canBeNegativeDividend()) {
if (mir->isTruncated()) {
// -0.0|0 == 0
} else {
MOZ_ASSERT(mir->fallible());
// See if X < 0
masm.as_cmp(output, Imm8(0));
masm.ma_b(&done, Assembler::NotEqual);
masm.as_cmp(lhs, Imm8(0));
bailoutIf(Assembler::Signed, ins->snapshot());
}
}
masm.bind(&done);
}
void CodeGenerator::visitSoftModI(LSoftModI* ins) {
// Extract the registers from this instruction.
Register lhs = ToRegister(ins->lhs());
Register rhs = ToRegister(ins->rhs());
Register output = ToRegister(ins->output());
Register callTemp = ToRegister(ins->callTemp());
MMod* mir = ins->mir();
Label done;
// Save the lhs in case we end up with a 0 that should be a -0.0 because lhs <
// 0.
MOZ_ASSERT(callTemp != lhs);
MOZ_ASSERT(callTemp != rhs);
masm.ma_mov(lhs, callTemp);
// Prevent INT_MIN % -1.
//
// |aeabi_idivmod| is allowed to return any arbitrary value when called with
// |(INT_MIN, -1)|, see "Run-time ABI for the ARM architecture manual". Most
// implementations perform a non-trapping signed integer division and
// return the expected result, i.e. INT_MIN. But since we can't rely on this
// behavior, handle this case separately here.
if (mir->canBeNegativeDividend()) {
{
ScratchRegisterScope scratch(masm);
// Sets EQ if lhs == INT_MIN
masm.ma_cmp(lhs, Imm32(INT_MIN), scratch);
// If EQ (LHS == INT_MIN), sets EQ if rhs == -1
masm.ma_cmp(rhs, Imm32(-1), scratch, Assembler::Equal);
}
if (mir->isTruncated()) {
// (INT_MIN % -1)|0 == 0
Label skip;
masm.ma_b(&skip, Assembler::NotEqual);
masm.ma_mov(Imm32(0), output);
masm.ma_b(&done);
masm.bind(&skip);
} else {
MOZ_ASSERT(mir->fallible());
bailoutIf(Assembler::Equal, ins->snapshot());
}
}
modICommon(mir, lhs, rhs, output, ins->snapshot(), done);
if (gen->compilingWasm()) {
masm.Push(InstanceReg);
int32_t framePushedAfterInstance = masm.framePushed();
masm.setupWasmABICall();
masm.passABIArg(lhs);
masm.passABIArg(rhs);
int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance;
masm.callWithABI(mir->bytecodeOffset(),
wasm::SymbolicAddress::aeabi_idivmod,
mozilla::Some(instanceOffset));
masm.Pop(InstanceReg);
} else {
using Fn = int64_t (*)(int, int);
masm.setupAlignedABICall();
masm.passABIArg(lhs);
masm.passABIArg(rhs);
masm.callWithABI<Fn, __aeabi_idivmod>(
MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckOther);
}
MOZ_ASSERT(r1 != output);
masm.move32(r1, output);
// If X%Y == 0 and X < 0, then we *actually* wanted to return -0.0
if (mir->canBeNegativeDividend()) {
if (mir->isTruncated()) {
// -0.0|0 == 0
} else {
MOZ_ASSERT(mir->fallible());
// See if X < 0
masm.as_cmp(output, Imm8(0));
masm.ma_b(&done, Assembler::NotEqual);
masm.as_cmp(callTemp, Imm8(0));
bailoutIf(Assembler::Signed, ins->snapshot());
}
}
masm.bind(&done);
}
void CodeGenerator::visitModPowTwoI(LModPowTwoI* ins) {
Register in = ToRegister(ins->getOperand(0));
Register out = ToRegister(ins->getDef(0));
MMod* mir = ins->mir();
Label fin;
// here.
masm.ma_mov(in, out, SetCC);
masm.ma_b(&fin, Assembler::Zero);
masm.as_rsb(out, out, Imm8(0), LeaveCC, Assembler::Signed);
{
ScratchRegisterScope scratch(masm);
masm.ma_and(Imm32((1 << ins->shift()) - 1), out, scratch);
}
masm.as_rsb(out, out, Imm8(0), SetCC, Assembler::Signed);
if (mir->canBeNegativeDividend()) {
if (!mir->isTruncated()) {
MOZ_ASSERT(mir->fallible());
bailoutIf(Assembler::Zero, ins->snapshot());
} else {
// -0|0 == 0
}
}
masm.bind(&fin);
}
void CodeGenerator::visitModMaskI(LModMaskI* ins) {
Register src = ToRegister(ins->getOperand(0));
Register dest = ToRegister(ins->getDef(0));
Register tmp1 = ToRegister(ins->getTemp(0));
Register tmp2 = ToRegister(ins->getTemp(1));
MMod* mir = ins->mir();
ScratchRegisterScope scratch(masm);
SecondScratchRegisterScope scratch2(masm);
masm.ma_mod_mask(src, dest, tmp1, tmp2, scratch, scratch2, ins->shift());
if (mir->canBeNegativeDividend()) {
if (!mir->isTruncated()) {
MOZ_ASSERT(mir->fallible());
bailoutIf(Assembler::Zero, ins->snapshot());
} else {
// -0|0 == 0
}
}
}
void CodeGeneratorARM::emitBigIntDiv(LBigIntDiv* ins, Register dividend,
Register divisor, Register output,
Label* fail) {
// Callers handle division by zero and integer overflow.
if (HasIDIV()) {
masm.ma_sdiv(dividend, divisor, /* result= */ dividend);
// Create and return the result.
masm.newGCBigInt(output, divisor, initialBigIntHeap(), fail);
masm.initializeBigInt(output, dividend);
return;
}
// idivmod returns the quotient in r0, and the remainder in r1.
MOZ_ASSERT(dividend == r0);
MOZ_ASSERT(divisor == r1);
LiveRegisterSet volatileRegs = liveVolatileRegs(ins);
volatileRegs.takeUnchecked(dividend);
volatileRegs.takeUnchecked(divisor);
volatileRegs.takeUnchecked(output);
masm.PushRegsInMask(volatileRegs);
using Fn = int64_t (*)(int, int);
masm.setupUnalignedABICall(output);
masm.passABIArg(dividend);
masm.passABIArg(divisor);
masm.callWithABI<Fn, __aeabi_idivmod>(MoveOp::GENERAL,
CheckUnsafeCallWithABI::DontCheckOther);
masm.PopRegsInMask(volatileRegs);
// Create and return the result.
masm.newGCBigInt(output, divisor, initialBigIntHeap(), fail);
masm.initializeBigInt(output, dividend);
}
void CodeGeneratorARM::emitBigIntMod(LBigIntMod* ins, Register dividend,
Register divisor, Register output,
Label* fail) {
// Callers handle division by zero and integer overflow.
if (HasIDIV()) {
{
ScratchRegisterScope scratch(masm);
masm.ma_smod(dividend, divisor, /* result= */ dividend, scratch);
}
// Create and return the result.
masm.newGCBigInt(output, divisor, initialBigIntHeap(), fail);
masm.initializeBigInt(output, dividend);
return;
}
// idivmod returns the quotient in r0, and the remainder in r1.
MOZ_ASSERT(dividend == r0);
MOZ_ASSERT(divisor == r1);
LiveRegisterSet volatileRegs = liveVolatileRegs(ins);
volatileRegs.takeUnchecked(dividend);
volatileRegs.takeUnchecked(divisor);
volatileRegs.takeUnchecked(output);
masm.PushRegsInMask(volatileRegs);
using Fn = int64_t (*)(int, int);
masm.setupUnalignedABICall(output);
masm.passABIArg(dividend);
masm.passABIArg(divisor);
masm.callWithABI<Fn, __aeabi_idivmod>(MoveOp::GENERAL,
CheckUnsafeCallWithABI::DontCheckOther);
masm.PopRegsInMask(volatileRegs);
// Create and return the result.
masm.newGCBigInt(output, dividend, initialBigIntHeap(), fail);
masm.initializeBigInt(output, divisor);
}
void CodeGenerator::visitBitNotI(LBitNotI* ins) {
const LAllocation* input = ins->getOperand(0);
const LDefinition* dest = ins->getDef(0);
// This will not actually be true on arm. We can not an imm8m in order to
// get a wider range of numbers
MOZ_ASSERT(!input->isConstant());
masm.ma_mvn(ToRegister(input), ToRegister(dest));
}
void CodeGenerator::visitBitOpI(LBitOpI* ins) {
const LAllocation* lhs = ins->getOperand(0);
const LAllocation* rhs = ins->getOperand(1);
const LDefinition* dest = ins->getDef(0);
ScratchRegisterScope scratch(masm);
// All of these bitops should be either imm32's, or integer registers.
switch (ins->bitop()) {
case JSOp::BitOr:
if (rhs->isConstant()) {
masm.ma_orr(Imm32(ToInt32(rhs)), ToRegister(lhs), ToRegister(dest),
scratch);
} else {
masm.ma_orr(ToRegister(rhs), ToRegister(lhs), ToRegister(dest));
}
break;
case JSOp::BitXor:
if (rhs->isConstant()) {
masm.ma_eor(Imm32(ToInt32(rhs)), ToRegister(lhs), ToRegister(dest),
scratch);
} else {
masm.ma_eor(ToRegister(rhs), ToRegister(lhs), ToRegister(dest));
}
break;
case JSOp::BitAnd:
if (rhs->isConstant()) {
masm.ma_and(Imm32(ToInt32(rhs)), ToRegister(lhs), ToRegister(dest),
scratch);
} else {
masm.ma_and(ToRegister(rhs), ToRegister(lhs), ToRegister(dest));
}
break;
default:
MOZ_CRASH("unexpected binary opcode");
}
}
void CodeGenerator::visitShiftI(LShiftI* ins) {
Register lhs = ToRegister(ins->lhs());
const LAllocation* rhs = ins->rhs();
Register dest = ToRegister(ins->output());
if (rhs->isConstant()) {
int32_t shift = ToInt32(rhs) & 0x1F;
switch (ins->bitop()) {
case JSOp::Lsh:
if (shift) {
masm.ma_lsl(Imm32(shift), lhs, dest);
} else {
masm.ma_mov(lhs, dest);
}
break;
case JSOp::Rsh:
if (shift) {
masm.ma_asr(Imm32(shift), lhs, dest);
} else {
masm.ma_mov(lhs, dest);
}
break;
case JSOp::Ursh:
if (shift) {
masm.ma_lsr(Imm32(shift), lhs, dest);
} else {
// x >>> 0 can overflow.
masm.ma_mov(lhs, dest);
if (ins->mir()->toUrsh()->fallible()) {
masm.as_cmp(dest, Imm8(0));
bailoutIf(Assembler::LessThan, ins->snapshot());
}
}
break;
default:
MOZ_CRASH("Unexpected shift op");
}
} else {
// The shift amounts should be AND'ed into the 0-31 range since arm
// shifts by the lower byte of the register (it will attempt to shift by
// 250 if you ask it to).
masm.as_and(dest, ToRegister(rhs), Imm8(0x1F));
switch (ins->bitop()) {
case JSOp::Lsh:
masm.ma_lsl(dest, lhs, dest);
break;
case JSOp::Rsh:
masm.ma_asr(dest, lhs, dest);
break;
case JSOp::Ursh:
masm.ma_lsr(dest, lhs, dest);
if (ins->mir()->toUrsh()->fallible()) {
// x >>> 0 can overflow.
masm.as_cmp(dest, Imm8(0));
bailoutIf(Assembler::LessThan, ins->snapshot());
}
break;
default:
MOZ_CRASH("Unexpected shift op");
}
}
}
void CodeGenerator::visitUrshD(LUrshD* ins) {
Register lhs = ToRegister(ins->lhs());
Register temp = ToRegister(ins->temp());
const LAllocation* rhs = ins->rhs();
FloatRegister out = ToFloatRegister(ins->output());
if (rhs->isConstant()) {
int32_t shift = ToInt32(rhs) & 0x1F;
if (shift) {
masm.ma_lsr(Imm32(shift), lhs, temp);
} else {
masm.ma_mov(lhs, temp);
}
} else {
masm.as_and(temp, ToRegister(rhs), Imm8(0x1F));
masm.ma_lsr(temp, lhs, temp);
}
masm.convertUInt32ToDouble(temp, out);
}
void CodeGenerator::visitClzI(LClzI* ins) {
Register input = ToRegister(ins->input());
Register output = ToRegister(ins->output());
masm.clz32(input, output, /* knownNotZero = */ false);
}
void CodeGenerator::visitCtzI(LCtzI* ins) {
Register input = ToRegister(ins->input());
Register output = ToRegister(ins->output());
masm.ctz32(input, output, /* knownNotZero = */ false);
}
void CodeGenerator::visitPopcntI(LPopcntI* ins) {
Register input = ToRegister(ins->input());
Register output = ToRegister(ins->output());
Register tmp = ToRegister(ins->temp0());
masm.popcnt32(input, output, tmp);
}
void CodeGenerator::visitPowHalfD(LPowHalfD* ins) {
FloatRegister input = ToFloatRegister(ins->input());
FloatRegister output = ToFloatRegister(ins->output());
ScratchDoubleScope scratch(masm);
Label done;
// Masm.pow(-Infinity, 0.5) == Infinity.
masm.loadConstantDouble(NegativeInfinity<double>(), scratch);
masm.compareDouble(input, scratch);
masm.ma_vneg(scratch, output, Assembler::Equal);
masm.ma_b(&done, Assembler::Equal);
// Math.pow(-0, 0.5) == 0 == Math.pow(0, 0.5).
// Adding 0 converts any -0 to 0.
masm.loadConstantDouble(0.0, scratch);
masm.ma_vadd(scratch, input, output);
masm.ma_vsqrt(output, output);
masm.bind(&done);
}
MoveOperand CodeGeneratorARM::toMoveOperand(LAllocation a) const {
if (a.isGeneralReg()) {
return MoveOperand(ToRegister(a));
}
if (a.isFloatReg()) {
return MoveOperand(ToFloatRegister(a));
}
MoveOperand::Kind kind = a.isStackArea() ? MoveOperand::Kind::EffectiveAddress
: MoveOperand::Kind::Memory;
Address addr = ToAddress(a);
MOZ_ASSERT((addr.offset & 3) == 0);
return MoveOperand(addr, kind);
}
class js::jit::OutOfLineTableSwitch
: public OutOfLineCodeBase<CodeGeneratorARM> {
MTableSwitch* mir_;
Vector<CodeLabel, 8, JitAllocPolicy> codeLabels_;
void accept(CodeGeneratorARM* codegen) override {
codegen->visitOutOfLineTableSwitch(this);
}
public:
OutOfLineTableSwitch(TempAllocator& alloc, MTableSwitch* mir)
: mir_(mir), codeLabels_(alloc) {}
MTableSwitch* mir() const { return mir_; }
bool addCodeLabel(CodeLabel label) { return codeLabels_.append(label); }
CodeLabel codeLabel(unsigned i) { return codeLabels_[i]; }
};
void CodeGeneratorARM::visitOutOfLineTableSwitch(OutOfLineTableSwitch* ool) {
MTableSwitch* mir = ool->mir();
size_t numCases = mir->numCases();
for (size_t i = 0; i < numCases; i++) {
LBlock* caseblock =
skipTrivialBlocks(mir->getCase(numCases - 1 - i))->lir();
Label* caseheader = caseblock->label();
uint32_t caseoffset = caseheader->offset();
// The entries of the jump table need to be absolute addresses and thus
// must be patched after codegen is finished.
CodeLabel cl = ool->codeLabel(i);
cl.target()->bind(caseoffset);
masm.addCodeLabel(cl);
}
}
void CodeGeneratorARM::emitTableSwitchDispatch(MTableSwitch* mir,
Register index, Register base) {
// The code generated by this is utter hax.
// The end result looks something like:
// SUBS index, input, #base
// RSBSPL index, index, #max
// LDRPL pc, pc, index lsl 2
// B default
// If the range of targets in N through M, we first subtract off the lowest
// case (N), which both shifts the arguments into the range 0 to (M - N)
// with and sets the MInus flag if the argument was out of range on the low
// end.
// Then we a reverse subtract with the size of the jump table, which will
// reverse the order of range (It is size through 0, rather than 0 through
// size). The main purpose of this is that we set the same flag as the lower
// bound check for the upper bound check. Lastly, we do this conditionally
// on the previous check succeeding.
// Then we conditionally load the pc offset by the (reversed) index (times
// the address size) into the pc, which branches to the correct case. NOTE:
// when we go to read the pc, the value that we get back is the pc of the
// current instruction *PLUS 8*. This means that ldr foo, [pc, +0] reads
// $pc+8. In other words, there is an empty word after the branch into the
// switch table before the table actually starts. Since the only other
// unhandled case is the default case (both out of range high and out of
// range low) I then insert a branch to default case into the extra slot,
// which ensures we don't attempt to execute the address table.
Label* defaultcase = skipTrivialBlocks(mir->getDefault())->lir()->label();
ScratchRegisterScope scratch(masm);
int32_t cases = mir->numCases();
// Lower value with low value.
masm.ma_sub(index, Imm32(mir->low()), index, scratch, SetCC);
masm.ma_rsb(index, Imm32(cases - 1), index, scratch, SetCC,
Assembler::NotSigned);
// Inhibit pools within the following sequence because we are indexing into
// a pc relative table. The region will have one instruction for ma_ldr, one
// for ma_b, and each table case takes one word.
AutoForbidPoolsAndNops afp(&masm, 1 + 1 + cases);
masm.ma_ldr(DTRAddr(pc, DtrRegImmShift(index, LSL, 2)), pc, Offset,
Assembler::NotSigned);
masm.ma_b(defaultcase);
// To fill in the CodeLabels for the case entries, we need to first generate
// the case entries (we don't yet know their offsets in the instruction
// stream).
OutOfLineTableSwitch* ool = new (alloc()) OutOfLineTableSwitch(alloc(), mir);
for (int32_t i = 0; i < cases; i++) {
CodeLabel cl;
masm.writeCodePointer(&cl);
masm.propagateOOM(ool->addCodeLabel(cl));
}
addOutOfLineCode(ool, mir);
}
void CodeGenerator::visitMathD(LMathD* math) {
FloatRegister src1 = ToFloatRegister(math->getOperand(0));
FloatRegister src2 = ToFloatRegister(math->getOperand(1));
FloatRegister output = ToFloatRegister(math->getDef(0));
switch (math->jsop()) {
case JSOp::Add:
masm.ma_vadd(src1, src2, output);
break;
case JSOp::Sub:
masm.ma_vsub(src1, src2, output);
break;
case JSOp::Mul:
masm.ma_vmul(src1, src2, output);
break;
case JSOp::Div:
masm.ma_vdiv(src1, src2, output);
break;
default:
MOZ_CRASH("unexpected opcode");
}
}
void CodeGenerator::visitMathF(LMathF* math) {
FloatRegister src1 = ToFloatRegister(math->getOperand(0));
FloatRegister src2 = ToFloatRegister(math->getOperand(1));
FloatRegister output = ToFloatRegister(math->getDef(0));
switch (math->jsop()) {
case JSOp::Add:
masm.ma_vadd_f32(src1, src2, output);
break;
case JSOp::Sub:
masm.ma_vsub_f32(src1, src2, output);
break;
case JSOp::Mul:
masm.ma_vmul_f32(src1, src2, output);
break;
case JSOp::Div:
masm.ma_vdiv_f32(src1, src2, output);
break;
default:
MOZ_CRASH("unexpected opcode");
}
}
void CodeGenerator::visitTruncateDToInt32(LTruncateDToInt32* ins) {
emitTruncateDouble(ToFloatRegister(ins->input()), ToRegister(ins->output()),
ins->mir());
}
void CodeGenerator::visitWasmBuiltinTruncateDToInt32(
LWasmBuiltinTruncateDToInt32* ins) {
emitTruncateDouble(ToFloatRegister(ins->getOperand(0)),
ToRegister(ins->getDef(0)), ins->mir());
}
void CodeGenerator::visitTruncateFToInt32(LTruncateFToInt32* ins) {
emitTruncateFloat32(ToFloatRegister(ins->input()), ToRegister(ins->output()),
ins->mir());
}
void CodeGenerator::visitWasmBuiltinTruncateFToInt32(
LWasmBuiltinTruncateFToInt32* ins) {
emitTruncateFloat32(ToFloatRegister(ins->getOperand(0)),
ToRegister(ins->getDef(0)), ins->mir());
}
ValueOperand CodeGeneratorARM::ToValue(LInstruction* ins, size_t pos) {
Register typeReg = ToRegister(ins->getOperand(pos + TYPE_INDEX));
Register payloadReg = ToRegister(ins->getOperand(pos + PAYLOAD_INDEX));
return ValueOperand(typeReg, payloadReg);
}
ValueOperand CodeGeneratorARM::ToTempValue(LInstruction* ins, size_t pos) {
Register typeReg = ToRegister(ins->getTemp(pos + TYPE_INDEX));
Register payloadReg = ToRegister(ins->getTemp(pos + PAYLOAD_INDEX));
return ValueOperand(typeReg, payloadReg);
}
void CodeGenerator::visitValue(LValue* value) {
const ValueOperand out = ToOutValue(value);
masm.moveValue(value->value(), out);
}
void CodeGenerator::visitBox(LBox* box) {
const LDefinition* type = box->getDef(TYPE_INDEX);
MOZ_ASSERT(!box->getOperand(0)->isConstant());
// On arm, the input operand and the output payload have the same virtual
// register. All that needs to be written is the type tag for the type
// definition.
masm.ma_mov(Imm32(MIRTypeToTag(box->type())), ToRegister(type));
}
void CodeGenerator::visitBoxFloatingPoint(LBoxFloatingPoint* box) {
const AnyRegister in = ToAnyRegister(box->getOperand(0));
const ValueOperand out = ToOutValue(box);
masm.moveValue(TypedOrValueRegister(box->type(), in), out);
}
void CodeGenerator::visitUnbox(LUnbox* unbox) {
// Note that for unbox, the type and payload indexes are switched on the
// inputs.
MUnbox* mir = unbox->mir();
Register type = ToRegister(unbox->type());
Register payload = ToRegister(unbox->payload());
Register output = ToRegister(unbox->output());
mozilla::Maybe<ScratchRegisterScope> scratch;
scratch.emplace(masm);
JSValueTag tag = MIRTypeToTag(mir->type());
if (mir->fallible()) {
masm.ma_cmp(type, Imm32(tag), *scratch);
bailoutIf(Assembler::NotEqual, unbox->snapshot());
} else {
#ifdef DEBUG
Label ok;
masm.ma_cmp(type, Imm32(tag), *scratch);
masm.ma_b(&ok, Assembler::Equal);
scratch.reset();
masm.assumeUnreachable("Infallible unbox type mismatch");
masm.bind(&ok);
#endif
}
// Note: If spectreValueMasking is disabled, then this instruction will
// default to a no-op as long as the lowering allocate the same register for
// the output and the payload.
masm.unboxNonDouble(ValueOperand(type, payload), output,
ValueTypeFromMIRType(mir->type()));
}
void CodeGenerator::visitDouble(LDouble* ins) {
const LDefinition* out = ins->getDef(0);
masm.loadConstantDouble(ins->value(), ToFloatRegister(out));
}
void CodeGenerator::visitFloat32(LFloat32* ins) {
const LDefinition* out = ins->getDef(0);
masm.loadConstantFloat32(ins->value(), ToFloatRegister(out));
}
void CodeGeneratorARM::splitTagForTest(const ValueOperand& value,
ScratchTagScope& tag) {
MOZ_ASSERT(value.typeReg() == tag);
}
void CodeGenerator::visitTestDAndBranch(LTestDAndBranch* test) {
const LAllocation* opd = test->input();
masm.ma_vcmpz(ToFloatRegister(opd));
masm.as_vmrs(pc);
MBasicBlock* ifTrue = test->ifTrue();
MBasicBlock* ifFalse = test->ifFalse();
// If the compare set the 0 bit, then the result is definitely false.
jumpToBlock(ifFalse, Assembler::Zero);
// It is also false if one of the operands is NAN, which is shown as
// Overflow.
jumpToBlock(ifFalse, Assembler::Overflow);
jumpToBlock(ifTrue);
}
void CodeGenerator::visitTestFAndBranch(LTestFAndBranch* test) {
const LAllocation* opd = test->input();
masm.ma_vcmpz_f32(ToFloatRegister(opd));
masm.as_vmrs(pc);
MBasicBlock* ifTrue = test->ifTrue();
MBasicBlock* ifFalse = test->ifFalse();
// If the compare set the 0 bit, then the result is definitely false.
jumpToBlock(ifFalse, Assembler::Zero);
// It is also false if one of the operands is NAN, which is shown as
// Overflow.
jumpToBlock(ifFalse, Assembler::Overflow);
jumpToBlock(ifTrue);
}
void CodeGenerator::visitCompareD(LCompareD* comp) {
FloatRegister lhs = ToFloatRegister(comp->left());
FloatRegister rhs = ToFloatRegister(comp->right());
Assembler::DoubleCondition cond = JSOpToDoubleCondition(comp->mir()->jsop());
masm.compareDouble(lhs, rhs);
masm.emitSet(Assembler::ConditionFromDoubleCondition(cond),
ToRegister(comp->output()));
}
void CodeGenerator::visitCompareF(LCompareF* comp) {
FloatRegister lhs = ToFloatRegister(comp->left());
FloatRegister rhs = ToFloatRegister(comp->right());
Assembler::DoubleCondition cond = JSOpToDoubleCondition(comp->mir()->jsop());
masm.compareFloat(lhs, rhs);
masm.emitSet(Assembler::ConditionFromDoubleCondition(cond),
ToRegister(comp->output()));
}
void CodeGenerator::visitCompareDAndBranch(LCompareDAndBranch* comp) {
FloatRegister lhs = ToFloatRegister(comp->left());
FloatRegister rhs = ToFloatRegister(comp->right());
Assembler::DoubleCondition cond =
JSOpToDoubleCondition(comp->cmpMir()->jsop());
masm.compareDouble(lhs, rhs);
emitBranch(Assembler::ConditionFromDoubleCondition(cond), comp->ifTrue(),
comp->ifFalse());
}
void CodeGenerator::visitCompareFAndBranch(LCompareFAndBranch* comp) {
FloatRegister lhs = ToFloatRegister(comp->left());
FloatRegister rhs = ToFloatRegister(comp->right());
Assembler::DoubleCondition cond =
JSOpToDoubleCondition(comp->cmpMir()->jsop());
masm.compareFloat(lhs, rhs);
emitBranch(Assembler::ConditionFromDoubleCondition(cond), comp->ifTrue(),
comp->ifFalse());
}
void CodeGenerator::visitBitAndAndBranch(LBitAndAndBranch* baab) {
// LBitAndAndBranch only represents single-word ANDs, hence it can't be
// 64-bit here.
MOZ_ASSERT(!baab->is64());
Register regL = ToRegister(baab->left());
if (baab->right()->isConstant()) {
ScratchRegisterScope scratch(masm);
masm.ma_tst(regL, Imm32(ToInt32(baab->right())), scratch);
} else {
masm.ma_tst(regL, ToRegister(baab->right()));
}
emitBranch(baab->cond(), baab->ifTrue(), baab->ifFalse());
}
void CodeGenerator::visitWasmUint32ToDouble(LWasmUint32ToDouble* lir) {
masm.convertUInt32ToDouble(ToRegister(lir->input()),
ToFloatRegister(lir->output()));
}
void CodeGenerator::visitWasmUint32ToFloat32(LWasmUint32ToFloat32* lir) {
masm.convertUInt32ToFloat32(ToRegister(lir->input()),
ToFloatRegister(lir->output()));
}
void CodeGenerator::visitNotI(LNotI* ins) {
// It is hard to optimize !x, so just do it the basic way for now.
masm.as_cmp(ToRegister(ins->input()), Imm8(0));
masm.emitSet(Assembler::Equal, ToRegister(ins->output()));
}
void CodeGenerator::visitNotI64(LNotI64* lir) {
Register64 input = ToRegister64(lir->getInt64Operand(0));
Register output = ToRegister(lir->output());
masm.ma_orr(input.low, input.high, output);
masm.as_cmp(output, Imm8(0));
masm.emitSet(Assembler::Equal, output);
}
void CodeGenerator::visitNotD(LNotD* ins) {
// Since this operation is not, we want to set a bit if the double is
// falsey, which means 0.0, -0.0 or NaN. When comparing with 0, an input of
// 0 will set the Z bit (30) and NaN will set the V bit (28) of the APSR.
FloatRegister opd = ToFloatRegister(ins->input());
Register dest = ToRegister(ins->output());
// Do the compare.
masm.ma_vcmpz(opd);
// TODO There are three variations here to compare performance-wise.
bool nocond = true;
if (nocond) {
// Load the value into the dest register.
masm.as_vmrs(dest);
masm.ma_lsr(Imm32(28), dest, dest);
// 28 + 2 = 30
masm.ma_alu(dest, lsr(dest, 2), dest, OpOrr);
masm.as_and(dest, dest, Imm8(1));
} else {
masm.as_vmrs(pc);
masm.ma_mov(Imm32(0), dest);
masm.ma_mov(Imm32(1), dest, Assembler::Equal);
masm.ma_mov(Imm32(1), dest, Assembler::Overflow);
}
}
void CodeGenerator::visitNotF(LNotF* ins) {
// Since this operation is not, we want to set a bit if the double is
// falsey, which means 0.0, -0.0 or NaN. When comparing with 0, an input of
// 0 will set the Z bit (30) and NaN will set the V bit (28) of the APSR.
FloatRegister opd = ToFloatRegister(ins->input());
Register dest = ToRegister(ins->output());
// Do the compare.
masm.ma_vcmpz_f32(opd);
// TODO There are three variations here to compare performance-wise.
bool nocond = true;
if (nocond) {
// Load the value into the dest register.
masm.as_vmrs(dest);
masm.ma_lsr(Imm32(28), dest, dest);
// 28 + 2 = 30
masm.ma_alu(dest, lsr(dest, 2), dest, OpOrr);
masm.as_and(dest, dest, Imm8(1));
} else {
masm.as_vmrs(pc);
masm.ma_mov(Imm32(0), dest);
masm.ma_mov(Imm32(1), dest, Assembler::Equal);
masm.ma_mov(Imm32(1), dest, Assembler::Overflow);
}
}
void CodeGeneratorARM::generateInvalidateEpilogue() {
// Ensure that there is enough space in the buffer for the OsiPoint patching
// to occur. Otherwise, we could overwrite the invalidation epilogue.
for (size_t i = 0; i < sizeof(void*); i += Assembler::NopSize()) {
masm.nop();
}
masm.bind(&invalidate_);
// Push the return address of the point that we bailed out at onto the stack.
masm.Push(lr);
// Push the Ion script onto the stack (when we determine what that pointer
// is).
invalidateEpilogueData_ = masm.pushWithPatch(ImmWord(uintptr_t(-1)));
// Jump to the invalidator which will replace the current frame.
TrampolinePtr thunk = gen->jitRuntime()->getInvalidationThunk();
masm.jump(thunk);
}
void CodeGenerator::visitCompareExchangeTypedArrayElement(
LCompareExchangeTypedArrayElement* lir) {
Register elements = ToRegister(lir->elements());
AnyRegister output = ToAnyRegister(lir->output());
Register temp =
lir->temp()->isBogusTemp() ? InvalidReg : ToRegister(lir->temp());
Register oldval = ToRegister(lir->oldval());
Register newval = ToRegister(lir->newval());
Scalar::Type arrayType = lir->mir()->arrayType();
if (lir->index()->isConstant()) {
Address dest = ToAddress(elements, lir->index(), arrayType);
masm.compareExchangeJS(arrayType, Synchronization::Full(), dest, oldval,
newval, temp, output);
} else {
BaseIndex dest(elements, ToRegister(lir->index()),
ScaleFromScalarType(arrayType));
masm.compareExchangeJS(arrayType, Synchronization::Full(), dest, oldval,
newval, temp, output);
}
}
void CodeGenerator::visitAtomicExchangeTypedArrayElement(
LAtomicExchangeTypedArrayElement* lir) {
Register elements = ToRegister(lir->elements());
AnyRegister output = ToAnyRegister(lir->output());
Register temp =
lir->temp()->isBogusTemp() ? InvalidReg : ToRegister(lir->temp());
Register value = ToRegister(lir->value());
Scalar::Type arrayType = lir->mir()->arrayType();
if (lir->index()->isConstant()) {
Address dest = ToAddress(elements, lir->index(), arrayType);
masm.atomicExchangeJS(arrayType, Synchronization::Full(), dest, value, temp,
output);
} else {
BaseIndex dest(elements, ToRegister(lir->index()),
ScaleFromScalarType(arrayType));
masm.atomicExchangeJS(arrayType, Synchronization::Full(), dest, value, temp,
output);