Source code

Revision control

Copy as Markdown

Other Tools

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: set ts=8 sts=2 et sw=2 tw=80:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef jit_x86_shared_MacroAssembler_x86_shared_inl_h
#define jit_x86_shared_MacroAssembler_x86_shared_inl_h
#include "jit/x86-shared/MacroAssembler-x86-shared.h"
#include "mozilla/MathAlgorithms.h"
namespace js {
namespace jit {
//{{{ check_macroassembler_style
// ===============================================================
// Move instructions
void MacroAssembler::moveFloat16ToGPR(FloatRegister src, Register dest) {
vmovd(src, dest);
// Ensure the hi-word is zeroed.
movzwl(dest, dest);
}
void MacroAssembler::moveGPRToFloat16(Register src, FloatRegister dest) {
// Ensure the hi-word is zeroed.
movzwl(src, src);
vmovd(src, dest);
}
void MacroAssembler::moveFloat32ToGPR(FloatRegister src, Register dest) {
vmovd(src, dest);
}
void MacroAssembler::moveGPRToFloat32(Register src, FloatRegister dest) {
vmovd(src, dest);
}
void MacroAssembler::move8ZeroExtend(Register src, Register dest) {
movzbl(src, dest);
}
void MacroAssembler::move8SignExtend(Register src, Register dest) {
movsbl(src, dest);
}
void MacroAssembler::move16SignExtend(Register src, Register dest) {
movswl(src, dest);
}
void MacroAssembler::loadAbiReturnAddress(Register dest) {
loadPtr(Address(getStackPointer(), 0), dest);
}
// ===============================================================
// Logical instructions
void MacroAssembler::not32(Register reg) { notl(reg); }
void MacroAssembler::and32(Register src, Register dest) { andl(src, dest); }
void MacroAssembler::and32(Imm32 imm, Register dest) { andl(imm, dest); }
void MacroAssembler::and32(Imm32 imm, const Address& dest) {
andl(imm, Operand(dest));
}
void MacroAssembler::and32(const Address& src, Register dest) {
andl(Operand(src), dest);
}
void MacroAssembler::or32(Register src, Register dest) { orl(src, dest); }
void MacroAssembler::or32(Imm32 imm, Register dest) { orl(imm, dest); }
void MacroAssembler::or32(Imm32 imm, const Address& dest) {
orl(imm, Operand(dest));
}
void MacroAssembler::xor32(Register src, Register dest) { xorl(src, dest); }
void MacroAssembler::xor32(Imm32 imm, Register dest) { xorl(imm, dest); }
void MacroAssembler::xor32(Imm32 imm, const Address& dest) {
xorl(imm, Operand(dest));
}
void MacroAssembler::xor32(const Address& src, Register dest) {
xorl(Operand(src), dest);
}
void MacroAssembler::clz32(Register src, Register dest, bool knownNotZero) {
if (AssemblerX86Shared::HasLZCNT()) {
lzcntl(src, dest);
return;
}
bsrl(src, dest);
if (!knownNotZero) {
// If the source is zero then bsrl leaves garbage in the destination.
Label nonzero;
j(Assembler::NonZero, &nonzero);
movl(Imm32(0x3F), dest);
bind(&nonzero);
}
xorl(Imm32(0x1F), dest);
}
void MacroAssembler::ctz32(Register src, Register dest, bool knownNotZero) {
if (AssemblerX86Shared::HasBMI1()) {
tzcntl(src, dest);
return;
}
bsfl(src, dest);
if (!knownNotZero) {
Label nonzero;
j(Assembler::NonZero, &nonzero);
movl(Imm32(32), dest);
bind(&nonzero);
}
}
void MacroAssembler::popcnt32(Register input, Register output, Register tmp) {
if (AssemblerX86Shared::HasPOPCNT()) {
popcntl(input, output);
return;
}
MOZ_ASSERT(tmp != InvalidReg);
// Equivalent to mozilla::CountPopulation32()
movl(input, tmp);
if (input != output) {
movl(input, output);
}
shrl(Imm32(1), output);
andl(Imm32(0x55555555), output);
subl(output, tmp);
movl(tmp, output);
andl(Imm32(0x33333333), output);
shrl(Imm32(2), tmp);
andl(Imm32(0x33333333), tmp);
addl(output, tmp);
movl(tmp, output);
shrl(Imm32(4), output);
addl(tmp, output);
andl(Imm32(0xF0F0F0F), output);
imull(Imm32(0x1010101), output, output);
shrl(Imm32(24), output);
}
// ===============================================================
// Swap instructions
void MacroAssembler::byteSwap16SignExtend(Register reg) {
rolw(Imm32(8), reg);
movswl(reg, reg);
}
void MacroAssembler::byteSwap16ZeroExtend(Register reg) {
rolw(Imm32(8), reg);
movzwl(reg, reg);
}
void MacroAssembler::byteSwap32(Register reg) { bswapl(reg); }
// ===============================================================
// Arithmetic instructions
void MacroAssembler::add32(Register src, Register dest) { addl(src, dest); }
void MacroAssembler::add32(Imm32 imm, Register dest) { addl(imm, dest); }
void MacroAssembler::add32(Imm32 imm, Register src, Register dest) {
leal(Operand(src, imm.value), dest);
}
void MacroAssembler::add32(Imm32 imm, const Address& dest) {
addl(imm, Operand(dest));
}
void MacroAssembler::add32(Imm32 imm, const AbsoluteAddress& dest) {
addl(imm, Operand(dest));
}
void MacroAssembler::addFloat32(FloatRegister src, FloatRegister dest) {
vaddss(src, dest, dest);
}
void MacroAssembler::addDouble(FloatRegister src, FloatRegister dest) {
vaddsd(src, dest, dest);
}
void MacroAssembler::sub32(Register src, Register dest) { subl(src, dest); }
void MacroAssembler::sub32(Imm32 imm, Register dest) { subl(imm, dest); }
void MacroAssembler::sub32(const Address& src, Register dest) {
subl(Operand(src), dest);
}
void MacroAssembler::subDouble(FloatRegister src, FloatRegister dest) {
vsubsd(src, dest, dest);
}
void MacroAssembler::subFloat32(FloatRegister src, FloatRegister dest) {
vsubss(src, dest, dest);
}
void MacroAssembler::mul32(Register rhs, Register srcDest) {
imull(rhs, srcDest);
}
void MacroAssembler::mul32(Imm32 imm, Register srcDest) { imull(imm, srcDest); }
void MacroAssembler::mulFloat32(FloatRegister src, FloatRegister dest) {
vmulss(src, dest, dest);
}
void MacroAssembler::mulDouble(FloatRegister src, FloatRegister dest) {
vmulsd(src, dest, dest);
}
void MacroAssembler::quotient32(Register rhs, Register srcDest,
Register tempEdx, bool isUnsigned) {
MOZ_ASSERT(srcDest == eax && tempEdx == edx);
// Sign extend eax into edx to make (edx:eax): idiv/udiv are 64-bit.
if (isUnsigned) {
mov(ImmWord(0), edx);
udiv(rhs);
} else {
cdq();
idiv(rhs);
}
}
void MacroAssembler::remainder32(Register rhs, Register srcDest,
Register tempEdx, bool isUnsigned) {
MOZ_ASSERT(srcDest == eax && tempEdx == edx);
// Sign extend eax into edx to make (edx:eax): idiv/udiv are 64-bit.
if (isUnsigned) {
mov(ImmWord(0), edx);
udiv(rhs);
} else {
cdq();
idiv(rhs);
}
mov(edx, eax);
}
void MacroAssembler::divFloat32(FloatRegister src, FloatRegister dest) {
vdivss(src, dest, dest);
}
void MacroAssembler::divDouble(FloatRegister src, FloatRegister dest) {
vdivsd(src, dest, dest);
}
void MacroAssembler::neg32(Register reg) { negl(reg); }
void MacroAssembler::negateFloat(FloatRegister reg) {
ScratchFloat32Scope scratch(*this);
vpcmpeqw(Operand(scratch), scratch, scratch);
vpsllq(Imm32(31), scratch, scratch);
// XOR the float in a float register with -0.0.
vxorps(scratch, reg, reg); // s ^ 0x80000000
}
void MacroAssembler::negateDouble(FloatRegister reg) {
// From MacroAssemblerX86Shared::maybeInlineDouble
ScratchDoubleScope scratch(*this);
vpcmpeqw(Operand(scratch), scratch, scratch);
vpsllq(Imm32(63), scratch, scratch);
// XOR the float in a float register with -0.0.
vxorpd(scratch, reg, reg); // s ^ 0x80000000000000
}
void MacroAssembler::abs32(Register src, Register dest) {
if (src != dest) {
move32(src, dest);
}
Label positive;
branchTest32(Assembler::NotSigned, dest, dest, &positive);
neg32(dest);
bind(&positive);
}
void MacroAssembler::absFloat32(FloatRegister src, FloatRegister dest) {
ScratchFloat32Scope scratch(*this);
loadConstantFloat32(mozilla::SpecificNaN<float>(
0, mozilla::FloatingPoint<float>::kSignificandBits),
scratch);
vandps(scratch, src, dest);
}
void MacroAssembler::absDouble(FloatRegister src, FloatRegister dest) {
ScratchDoubleScope scratch(*this);
loadConstantDouble(mozilla::SpecificNaN<double>(
0, mozilla::FloatingPoint<double>::kSignificandBits),
scratch);
vandpd(scratch, src, dest);
}
void MacroAssembler::sqrtFloat32(FloatRegister src, FloatRegister dest) {
vsqrtss(src, dest, dest);
}
void MacroAssembler::sqrtDouble(FloatRegister src, FloatRegister dest) {
vsqrtsd(src, dest, dest);
}
void MacroAssembler::minFloat32(FloatRegister other, FloatRegister srcDest,
bool handleNaN) {
minMaxFloat32(srcDest, other, handleNaN, false);
}
void MacroAssembler::minDouble(FloatRegister other, FloatRegister srcDest,
bool handleNaN) {
minMaxDouble(srcDest, other, handleNaN, false);
}
void MacroAssembler::maxFloat32(FloatRegister other, FloatRegister srcDest,
bool handleNaN) {
minMaxFloat32(srcDest, other, handleNaN, true);
}
void MacroAssembler::maxDouble(FloatRegister other, FloatRegister srcDest,
bool handleNaN) {
minMaxDouble(srcDest, other, handleNaN, true);
}
// ===============================================================
// Rotation instructions
void MacroAssembler::rotateLeft(Imm32 count, Register input, Register dest) {
MOZ_ASSERT(input == dest, "defineReuseInput");
count.value &= 0x1f;
if (count.value) {
roll(count, input);
}
}
void MacroAssembler::rotateLeft(Register count, Register input, Register dest) {
MOZ_ASSERT(input == dest, "defineReuseInput");
MOZ_ASSERT(count == ecx, "defineFixed(ecx)");
roll_cl(input);
}
void MacroAssembler::rotateRight(Imm32 count, Register input, Register dest) {
MOZ_ASSERT(input == dest, "defineReuseInput");
count.value &= 0x1f;
if (count.value) {
rorl(count, input);
}
}
void MacroAssembler::rotateRight(Register count, Register input,
Register dest) {
MOZ_ASSERT(input == dest, "defineReuseInput");
MOZ_ASSERT(count == ecx, "defineFixed(ecx)");
rorl_cl(input);
}
// ===============================================================
// Shift instructions
void MacroAssembler::lshift32(Register shift, Register srcDest) {
if (HasBMI2()) {
shlxl(srcDest, shift, srcDest);
return;
}
MOZ_ASSERT(shift == ecx);
shll_cl(srcDest);
}
void MacroAssembler::flexibleLshift32(Register shift, Register srcDest) {
if (HasBMI2()) {
shlxl(srcDest, shift, srcDest);
return;
}
if (shift == ecx) {
shll_cl(srcDest);
} else {
// Shift amount must be in ecx.
xchg(shift, ecx);
shll_cl(shift == srcDest ? ecx : srcDest == ecx ? shift : srcDest);
xchg(shift, ecx);
}
}
void MacroAssembler::rshift32(Register shift, Register srcDest) {
if (HasBMI2()) {
shrxl(srcDest, shift, srcDest);
return;
}
MOZ_ASSERT(shift == ecx);
shrl_cl(srcDest);
}
void MacroAssembler::flexibleRshift32(Register shift, Register srcDest) {
if (HasBMI2()) {
shrxl(srcDest, shift, srcDest);
return;
}
if (shift == ecx) {
shrl_cl(srcDest);
} else {
// Shift amount must be in ecx.
xchg(shift, ecx);
shrl_cl(shift == srcDest ? ecx : srcDest == ecx ? shift : srcDest);
xchg(shift, ecx);
}
}
void MacroAssembler::rshift32Arithmetic(Register shift, Register srcDest) {
if (HasBMI2()) {
sarxl(srcDest, shift, srcDest);
return;
}
MOZ_ASSERT(shift == ecx);
sarl_cl(srcDest);
}
void MacroAssembler::flexibleRshift32Arithmetic(Register shift,
Register srcDest) {
if (HasBMI2()) {
sarxl(srcDest, shift, srcDest);
return;
}
if (shift == ecx) {
sarl_cl(srcDest);
} else {
// Shift amount must be in ecx.
xchg(shift, ecx);
sarl_cl(shift == srcDest ? ecx : srcDest == ecx ? shift : srcDest);
xchg(shift, ecx);
}
}
void MacroAssembler::lshift32(Imm32 shift, Register srcDest) {
shll(shift, srcDest);
}
void MacroAssembler::rshift32(Imm32 shift, Register srcDest) {
shrl(shift, srcDest);
}
void MacroAssembler::rshift32Arithmetic(Imm32 shift, Register srcDest) {
sarl(shift, srcDest);
}
// ===============================================================
// Condition functions
void MacroAssembler::cmp8Set(Condition cond, Address lhs, Imm32 rhs,
Register dest) {
cmp8(lhs, rhs);
emitSet(cond, dest);
}
void MacroAssembler::cmp16Set(Condition cond, Address lhs, Imm32 rhs,
Register dest) {
cmp16(lhs, rhs);
emitSet(cond, dest);
}
template <typename T1, typename T2>
void MacroAssembler::cmp32Set(Condition cond, T1 lhs, T2 rhs, Register dest) {
cmp32(lhs, rhs);
emitSet(cond, dest);
}
// ===============================================================
// Branch instructions
void MacroAssembler::branch8(Condition cond, const Address& lhs, Imm32 rhs,
Label* label) {
cmp8(lhs, rhs);
j(cond, label);
}
void MacroAssembler::branch8(Condition cond, const BaseIndex& lhs, Register rhs,
Label* label) {
cmp8(Operand(lhs), rhs);
j(cond, label);
}
void MacroAssembler::branch16(Condition cond, const Address& lhs, Imm32 rhs,
Label* label) {
cmp16(lhs, rhs);
j(cond, label);
}
template <class L>
void MacroAssembler::branch32(Condition cond, Register lhs, Register rhs,
L label) {
cmp32(lhs, rhs);
j(cond, label);
}
template <class L>
void MacroAssembler::branch32(Condition cond, Register lhs, Imm32 rhs,
L label) {
cmp32(lhs, rhs);
j(cond, label);
}
void MacroAssembler::branch32(Condition cond, const Address& lhs, Register rhs,
Label* label) {
cmp32(Operand(lhs), rhs);
j(cond, label);
}
void MacroAssembler::branch32(Condition cond, const Address& lhs, Imm32 rhs,
Label* label) {
cmp32(Operand(lhs), rhs);
j(cond, label);
}
void MacroAssembler::branch32(Condition cond, const BaseIndex& lhs,
Register rhs, Label* label) {
cmp32(Operand(lhs), rhs);
j(cond, label);
}
void MacroAssembler::branch32(Condition cond, const BaseIndex& lhs, Imm32 rhs,
Label* label) {
cmp32(Operand(lhs), rhs);
j(cond, label);
}
void MacroAssembler::branch32(Condition cond, const Operand& lhs, Register rhs,
Label* label) {
cmp32(lhs, rhs);
j(cond, label);
}
void MacroAssembler::branch32(Condition cond, const Operand& lhs, Imm32 rhs,
Label* label) {
cmp32(lhs, rhs);
j(cond, label);
}
template <class L>
void MacroAssembler::branchPtr(Condition cond, Register lhs, Register rhs,
L label) {
cmpPtr(lhs, rhs);
j(cond, label);
}
void MacroAssembler::branchPtr(Condition cond, Register lhs, Imm32 rhs,
Label* label) {
branchPtrImpl(cond, lhs, rhs, label);
}
void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmPtr rhs,
Label* label) {
branchPtrImpl(cond, lhs, rhs, label);
}
void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmGCPtr rhs,
Label* label) {
branchPtrImpl(cond, lhs, rhs, label);
}
void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmWord rhs,
Label* label) {
branchPtrImpl(cond, lhs, rhs, label);
}
template <class L>
void MacroAssembler::branchPtr(Condition cond, const Address& lhs, Register rhs,
L label) {
branchPtrImpl(cond, lhs, rhs, label);
}
void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmPtr rhs,
Label* label) {
branchPtrImpl(cond, lhs, rhs, label);
}
void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmGCPtr rhs,
Label* label) {
branchPtrImpl(cond, lhs, rhs, label);
}
void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmWord rhs,
Label* label) {
branchPtrImpl(cond, lhs, rhs, label);
}
void MacroAssembler::branchPtr(Condition cond, const BaseIndex& lhs,
ImmWord rhs, Label* label) {
branchPtrImpl(cond, lhs, rhs, label);
}
void MacroAssembler::branchPtr(Condition cond, const BaseIndex& lhs,
Register rhs, Label* label) {
branchPtrImpl(cond, lhs, rhs, label);
}
template <typename T, typename S, typename L>
void MacroAssembler::branchPtrImpl(Condition cond, const T& lhs, const S& rhs,
L label) {
cmpPtr(Operand(lhs), rhs);
j(cond, label);
}
void MacroAssembler::branchFloat(DoubleCondition cond, FloatRegister lhs,
FloatRegister rhs, Label* label) {
compareFloat(cond, lhs, rhs);
if (cond == DoubleEqual) {
Label unordered;
j(Parity, &unordered);
j(Equal, label);
bind(&unordered);
return;
}
if (cond == DoubleNotEqualOrUnordered) {
j(NotEqual, label);
j(Parity, label);
return;
}
MOZ_ASSERT(!(cond & DoubleConditionBitSpecial));
j(ConditionFromDoubleCondition(cond), label);
}
void MacroAssembler::branchDouble(DoubleCondition cond, FloatRegister lhs,
FloatRegister rhs, Label* label) {
compareDouble(cond, lhs, rhs);
if (cond == DoubleEqual) {
Label unordered;
j(Parity, &unordered);
j(Equal, label);
bind(&unordered);
return;
}
if (cond == DoubleNotEqualOrUnordered) {
j(NotEqual, label);
j(Parity, label);
return;
}
MOZ_ASSERT(!(cond & DoubleConditionBitSpecial));
j(ConditionFromDoubleCondition(cond), label);
}
template <typename T>
void MacroAssembler::branchAdd32(Condition cond, T src, Register dest,
Label* label) {
addl(src, dest);
j(cond, label);
}
template <typename T>
void MacroAssembler::branchSub32(Condition cond, T src, Register dest,
Label* label) {
subl(src, dest);
j(cond, label);
}
template <typename T>
void MacroAssembler::branchMul32(Condition cond, T src, Register dest,
Label* label) {
mul32(src, dest);
j(cond, label);
}
template <typename T>
void MacroAssembler::branchRshift32(Condition cond, T src, Register dest,
Label* label) {
MOZ_ASSERT(cond == Zero || cond == NonZero);
rshift32(src, dest);
j(cond, label);
}
void MacroAssembler::branchNeg32(Condition cond, Register reg, Label* label) {
MOZ_ASSERT(cond == Overflow);
neg32(reg);
j(cond, label);
}
template <typename T>
void MacroAssembler::branchAddPtr(Condition cond, T src, Register dest,
Label* label) {
addPtr(src, dest);
j(cond, label);
}
template <typename T>
void MacroAssembler::branchSubPtr(Condition cond, T src, Register dest,
Label* label) {
subPtr(src, dest);
j(cond, label);
}
void MacroAssembler::branchMulPtr(Condition cond, Register src, Register dest,
Label* label) {
mulPtr(src, dest);
j(cond, label);
}
void MacroAssembler::decBranchPtr(Condition cond, Register lhs, Imm32 rhs,
Label* label) {
subPtr(rhs, lhs);
j(cond, label);
}
template <class L>
void MacroAssembler::branchTest32(Condition cond, Register lhs, Register rhs,
L label) {
MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
cond == NotSigned);
test32(lhs, rhs);
j(cond, label);
}
template <class L>
void MacroAssembler::branchTest32(Condition cond, Register lhs, Imm32 rhs,
L label) {
MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
cond == NotSigned);
test32(lhs, rhs);
j(cond, label);
}
void MacroAssembler::branchTest32(Condition cond, const Address& lhs, Imm32 rhs,
Label* label) {
MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
cond == NotSigned);
test32(Operand(lhs), rhs);
j(cond, label);
}
template <class L>
void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Register rhs,
L label) {
testPtr(lhs, rhs);
j(cond, label);
}
void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Imm32 rhs,
Label* label) {
testPtr(lhs, rhs);
j(cond, label);
}
void MacroAssembler::branchTestPtr(Condition cond, const Address& lhs,
Imm32 rhs, Label* label) {
testPtr(Operand(lhs), rhs);
j(cond, label);
}
void MacroAssembler::branchTestUndefined(Condition cond, Register tag,
Label* label) {
branchTestUndefinedImpl(cond, tag, label);
}
void MacroAssembler::branchTestUndefined(Condition cond, const Address& address,
Label* label) {
branchTestUndefinedImpl(cond, address, label);
}
void MacroAssembler::branchTestUndefined(Condition cond,
const BaseIndex& address,
Label* label) {
branchTestUndefinedImpl(cond, address, label);
}
void MacroAssembler::branchTestUndefined(Condition cond,
const ValueOperand& value,
Label* label) {
branchTestUndefinedImpl(cond, value, label);
}
template <typename T>
void MacroAssembler::branchTestUndefinedImpl(Condition cond, const T& t,
Label* label) {
cond = testUndefined(cond, t);
j(cond, label);
}
void MacroAssembler::branchTestInt32(Condition cond, Register tag,
Label* label) {
branchTestInt32Impl(cond, tag, label);
}
void MacroAssembler::branchTestInt32(Condition cond, const Address& address,
Label* label) {
branchTestInt32Impl(cond, address, label);
}
void MacroAssembler::branchTestInt32(Condition cond, const BaseIndex& address,
Label* label) {
branchTestInt32Impl(cond, address, label);
}
void MacroAssembler::branchTestInt32(Condition cond, const ValueOperand& value,
Label* label) {
branchTestInt32Impl(cond, value, label);
}
template <typename T>
void MacroAssembler::branchTestInt32Impl(Condition cond, const T& t,
Label* label) {
cond = testInt32(cond, t);
j(cond, label);
}
void MacroAssembler::branchTestInt32Truthy(bool truthy,
const ValueOperand& value,
Label* label) {
Condition cond = testInt32Truthy(truthy, value);
j(cond, label);
}
void MacroAssembler::branchTestDouble(Condition cond, Register tag,
Label* label) {
branchTestDoubleImpl(cond, tag, label);
}
void MacroAssembler::branchTestDouble(Condition cond, const Address& address,
Label* label) {
branchTestDoubleImpl(cond, address, label);
}
void MacroAssembler::branchTestDouble(Condition cond, const BaseIndex& address,
Label* label) {
branchTestDoubleImpl(cond, address, label);
}
void MacroAssembler::branchTestDouble(Condition cond, const ValueOperand& value,
Label* label) {
branchTestDoubleImpl(cond, value, label);
}
template <typename T>
void MacroAssembler::branchTestDoubleImpl(Condition cond, const T& t,
Label* label) {
cond = testDouble(cond, t);
j(cond, label);
}
void MacroAssembler::branchTestDoubleTruthy(bool truthy, FloatRegister reg,
Label* label) {
Condition cond = testDoubleTruthy(truthy, reg);
j(cond, label);
}
void MacroAssembler::branchTestNumber(Condition cond, Register tag,
Label* label) {
branchTestNumberImpl(cond, tag, label);
}
void MacroAssembler::branchTestNumber(Condition cond, const ValueOperand& value,
Label* label) {
branchTestNumberImpl(cond, value, label);
}
template <typename T>
void MacroAssembler::branchTestNumberImpl(Condition cond, const T& t,
Label* label) {
cond = testNumber(cond, t);
j(cond, label);
}
void MacroAssembler::branchTestBoolean(Condition cond, Register tag,
Label* label) {
branchTestBooleanImpl(cond, tag, label);
}
void MacroAssembler::branchTestBoolean(Condition cond, const Address& address,
Label* label) {
branchTestBooleanImpl(cond, address, label);
}
void MacroAssembler::branchTestBoolean(Condition cond, const BaseIndex& address,
Label* label) {
branchTestBooleanImpl(cond, address, label);
}
void MacroAssembler::branchTestBoolean(Condition cond,
const ValueOperand& value,
Label* label) {
branchTestBooleanImpl(cond, value, label);
}
template <typename T>
void MacroAssembler::branchTestBooleanImpl(Condition cond, const T& t,
Label* label) {
cond = testBoolean(cond, t);
j(cond, label);
}
void MacroAssembler::branchTestString(Condition cond, Register tag,
Label* label) {
branchTestStringImpl(cond, tag, label);
}
void MacroAssembler::branchTestString(Condition cond, const Address& address,
Label* label) {
branchTestStringImpl(cond, address, label);
}
void MacroAssembler::branchTestString(Condition cond, const BaseIndex& address,
Label* label) {
branchTestStringImpl(cond, address, label);
}
void MacroAssembler::branchTestString(Condition cond, const ValueOperand& value,
Label* label) {
branchTestStringImpl(cond, value, label);
}
template <typename T>
void MacroAssembler::branchTestStringImpl(Condition cond, const T& t,
Label* label) {
cond = testString(cond, t);
j(cond, label);
}
void MacroAssembler::branchTestStringTruthy(bool truthy,
const ValueOperand& value,
Label* label) {
Condition cond = testStringTruthy(truthy, value);
j(cond, label);
}
void MacroAssembler::branchTestSymbol(Condition cond, Register tag,
Label* label) {
branchTestSymbolImpl(cond, tag, label);
}
void MacroAssembler::branchTestSymbol(Condition cond, const Address& address,
Label* label) {
branchTestSymbolImpl(cond, address, label);
}
void MacroAssembler::branchTestSymbol(Condition cond, const BaseIndex& address,
Label* label) {
branchTestSymbolImpl(cond, address, label);
}
void MacroAssembler::branchTestSymbol(Condition cond, const ValueOperand& value,
Label* label) {
branchTestSymbolImpl(cond, value, label);
}
template <typename T>
void MacroAssembler::branchTestSymbolImpl(Condition cond, const T& t,
Label* label) {
cond = testSymbol(cond, t);
j(cond, label);
}
void MacroAssembler::branchTestBigInt(Condition cond, Register tag,
Label* label) {
branchTestBigIntImpl(cond, tag, label);
}
void MacroAssembler::branchTestBigInt(Condition cond, const Address& address,
Label* label) {
branchTestBigIntImpl(cond, address, label);
}
void MacroAssembler::branchTestBigInt(Condition cond, const BaseIndex& address,
Label* label) {
branchTestBigIntImpl(cond, address, label);
}
void MacroAssembler::branchTestBigInt(Condition cond, const ValueOperand& value,
Label* label) {
branchTestBigIntImpl(cond, value, label);
}
template <typename T>
void MacroAssembler::branchTestBigIntImpl(Condition cond, const T& t,
Label* label) {
cond = testBigInt(cond, t);
j(cond, label);
}
void MacroAssembler::branchTestBigIntTruthy(bool truthy,
const ValueOperand& value,
Label* label) {
Condition cond = testBigIntTruthy(truthy, value);
j(cond, label);
}
void MacroAssembler::branchTestNull(Condition cond, Register tag,
Label* label) {
branchTestNullImpl(cond, tag, label);
}
void MacroAssembler::branchTestNull(Condition cond, const Address& address,
Label* label) {
branchTestNullImpl(cond, address, label);
}
void MacroAssembler::branchTestNull(Condition cond, const BaseIndex& address,
Label* label) {
branchTestNullImpl(cond, address, label);
}
void MacroAssembler::branchTestNull(Condition cond, const ValueOperand& value,
Label* label) {
branchTestNullImpl(cond, value, label);
}
template <typename T>
void MacroAssembler::branchTestNullImpl(Condition cond, const T& t,
Label* label) {
cond = testNull(cond, t);
j(cond, label);
}
void MacroAssembler::branchTestObject(Condition cond, Register tag,
Label* label) {
branchTestObjectImpl(cond, tag, label);
}
void MacroAssembler::branchTestObject(Condition cond, const Address& address,
Label* label) {
branchTestObjectImpl(cond, address, label);
}
void MacroAssembler::branchTestObject(Condition cond, const BaseIndex& address,
Label* label) {
branchTestObjectImpl(cond, address, label);
}
void MacroAssembler::branchTestObject(Condition cond, const ValueOperand& value,
Label* label) {
branchTestObjectImpl(cond, value, label);
}
template <typename T>
void MacroAssembler::branchTestObjectImpl(Condition cond, const T& t,
Label* label) {
cond = testObject(cond, t);
j(cond, label);
}
void MacroAssembler::branchTestGCThing(Condition cond, const Address& address,
Label* label) {
branchTestGCThingImpl(cond, address, label);
}
void MacroAssembler::branchTestGCThing(Condition cond, const BaseIndex& address,
Label* label) {
branchTestGCThingImpl(cond, address, label);
}
void MacroAssembler::branchTestGCThing(Condition cond,
const ValueOperand& value,
Label* label) {
branchTestGCThingImpl(cond, value, label);
}
template <typename T>
void MacroAssembler::branchTestGCThingImpl(Condition cond, const T& t,
Label* label) {
cond = testGCThing(cond, t);
j(cond, label);
}
void MacroAssembler::branchTestPrimitive(Condition cond, Register tag,
Label* label) {
branchTestPrimitiveImpl(cond, tag, label);
}
void MacroAssembler::branchTestPrimitive(Condition cond,
const ValueOperand& value,
Label* label) {
branchTestPrimitiveImpl(cond, value, label);
}
template <typename T>
void MacroAssembler::branchTestPrimitiveImpl(Condition cond, const T& t,
Label* label) {
cond = testPrimitive(cond, t);
j(cond, label);
}
void MacroAssembler::branchTestMagic(Condition cond, Register tag,
Label* label) {
branchTestMagicImpl(cond, tag, label);
}
void MacroAssembler::branchTestMagic(Condition cond, const Address& address,
Label* label) {
branchTestMagicImpl(cond, address, label);
}
void MacroAssembler::branchTestMagic(Condition cond, const BaseIndex& address,
Label* label) {
branchTestMagicImpl(cond, address, label);
}
template <class L>
void MacroAssembler::branchTestMagic(Condition cond, const ValueOperand& value,
L label) {
branchTestMagicImpl(cond, value, label);
}
template <typename T, class L>
void MacroAssembler::branchTestMagicImpl(Condition cond, const T& t, L label) {
cond = testMagic(cond, t);
j(cond, label);
}
template <typename T>
void MacroAssembler::testNumberSet(Condition cond, const T& src,
Register dest) {
cond = testNumber(cond, src);
emitSet(cond, dest);
}
template <typename T>
void MacroAssembler::testBooleanSet(Condition cond, const T& src,
Register dest) {
cond = testBoolean(cond, src);
emitSet(cond, dest);
}
template <typename T>
void MacroAssembler::testStringSet(Condition cond, const T& src,
Register dest) {
cond = testString(cond, src);
emitSet(cond, dest);
}
template <typename T>
void MacroAssembler::testSymbolSet(Condition cond, const T& src,
Register dest) {
cond = testSymbol(cond, src);
emitSet(cond, dest);
}
template <typename T>
void MacroAssembler::testBigIntSet(Condition cond, const T& src,
Register dest) {
cond = testBigInt(cond, src);
emitSet(cond, dest);
}
void MacroAssembler::cmp32Move32(Condition cond, Register lhs, Imm32 rhs,
Register src, Register dest) {
cmp32(lhs, rhs);
cmovCCl(cond, src, dest);
}
void MacroAssembler::cmp32Move32(Condition cond, Register lhs, Register rhs,
Register src, Register dest) {
cmp32(lhs, rhs);
cmovCCl(cond, src, dest);
}
void MacroAssembler::cmp32Move32(Condition cond, Register lhs,
const Address& rhs, Register src,
Register dest) {
cmp32(lhs, Operand(rhs));
cmovCCl(cond, src, dest);
}
void MacroAssembler::cmp32Load32(Condition cond, Register lhs,
const Address& rhs, const Address& src,
Register dest) {
cmp32(lhs, Operand(rhs));
cmovCCl(cond, Operand(src), dest);
}
void MacroAssembler::cmp32Load32(Condition cond, Register lhs, Register rhs,
const Address& src, Register dest) {
cmp32(lhs, rhs);
cmovCCl(cond, Operand(src), dest);
}
void MacroAssembler::cmp32Load32(Condition cond, Register lhs, Imm32 rhs,
const Address& src, Register dest) {
cmp32(lhs, rhs);
cmovCCl(cond, Operand(src), dest);
}
void MacroAssembler::spectreZeroRegister(Condition cond, Register scratch,
Register dest) {
// Note: use movl instead of move32/xorl to ensure flags are not clobbered.
movl(Imm32(0), scratch);
spectreMovePtr(cond, scratch, dest);
}
// ========================================================================
// Memory access primitives.
FaultingCodeOffset MacroAssembler::storeUncanonicalizedDouble(
FloatRegister src, const Address& dest) {
FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
vmovsd(src, dest);
return fco;
}
FaultingCodeOffset MacroAssembler::storeUncanonicalizedDouble(
FloatRegister src, const BaseIndex& dest) {
FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
vmovsd(src, dest);
return fco;
}
FaultingCodeOffset MacroAssembler::storeUncanonicalizedDouble(
FloatRegister src, const Operand& dest) {
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
return storeUncanonicalizedDouble(src, dest.toAddress());
case Operand::MEM_SCALE:
return storeUncanonicalizedDouble(src, dest.toBaseIndex());
default:
MOZ_CRASH("unexpected operand kind");
}
}
template FaultingCodeOffset MacroAssembler::storeDouble(FloatRegister src,
const Operand& dest);
FaultingCodeOffset MacroAssembler::storeUncanonicalizedFloat32(
FloatRegister src, const Address& dest) {
FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
vmovss(src, dest);
return fco;
}
FaultingCodeOffset MacroAssembler::storeUncanonicalizedFloat32(
FloatRegister src, const BaseIndex& dest) {
FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
vmovss(src, dest);
return fco;
}
FaultingCodeOffset MacroAssembler::storeUncanonicalizedFloat32(
FloatRegister src, const Operand& dest) {
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
return storeUncanonicalizedFloat32(src, dest.toAddress());
case Operand::MEM_SCALE:
return storeUncanonicalizedFloat32(src, dest.toBaseIndex());
default:
MOZ_CRASH("unexpected operand kind");
}
}
template FaultingCodeOffset MacroAssembler::storeFloat32(FloatRegister src,
const Operand& dest);
FaultingCodeOffset MacroAssembler::storeUncanonicalizedFloat16(
FloatRegister src, const Address& dest, Register scratch) {
vmovd(src, scratch);
FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
movw(scratch, Operand(dest));
return fco;
}
FaultingCodeOffset MacroAssembler::storeUncanonicalizedFloat16(
FloatRegister src, const BaseIndex& dest, Register scratch) {
vmovd(src, scratch);
FaultingCodeOffset fco = FaultingCodeOffset(currentOffset());
movw(scratch, Operand(dest));
return fco;
}
void MacroAssembler::memoryBarrier(MemoryBarrierBits barrier) {
if (barrier & MembarStoreLoad) {
// This implementation follows Linux.
masm.mfence();
}
}
// ========================================================================
// Wasm SIMD
//
// Some parts of the masm API are currently agnostic as to the data's
// interpretation as int or float, despite the Intel architecture having
// separate functional units and sometimes penalizing type-specific instructions
// that operate on data in the "wrong" unit.
//
// For the time being, we always choose the integer interpretation when we are
// forced to choose blind, but whether that is right or wrong depends on the
// application. This applies to moveSimd128, loadConstantSimd128,
// loadUnalignedSimd128, and storeUnalignedSimd128, at least.
//
// SSE4.1 or better is assumed.
//
// The order of operations here follows the header file.
// Moves. See comments above regarding integer operation.
void MacroAssembler::moveSimd128(FloatRegister src, FloatRegister dest) {
MacroAssemblerX86Shared::moveSimd128Int(src, dest);
}
// Constants. See comments above regarding integer operation.
void MacroAssembler::loadConstantSimd128(const SimdConstant& v,
FloatRegister dest) {
if (v.isFloatingType()) {
loadConstantSimd128Float(v, dest);
} else {
loadConstantSimd128Int(v, dest);
}
}
// Splat
void MacroAssembler::splatX16(Register src, FloatRegister dest) {
MacroAssemblerX86Shared::splatX16(src, dest);
}
void MacroAssembler::splatX8(Register src, FloatRegister dest) {
MacroAssemblerX86Shared::splatX8(src, dest);
}
void MacroAssembler::splatX4(Register src, FloatRegister dest) {
MacroAssemblerX86Shared::splatX4(src, dest);
}
void MacroAssembler::splatX4(FloatRegister src, FloatRegister dest) {
MacroAssemblerX86Shared::splatX4(src, dest);
}
void MacroAssembler::splatX2(FloatRegister src, FloatRegister dest) {
MacroAssemblerX86Shared::splatX2(src, dest);
}
// Extract lane as scalar
void MacroAssembler::extractLaneInt8x16(uint32_t lane, FloatRegister src,
Register dest) {
MacroAssemblerX86Shared::extractLaneInt8x16(src, dest, lane,
SimdSign::Signed);
}
void MacroAssembler::unsignedExtractLaneInt8x16(uint32_t lane,
FloatRegister src,
Register dest) {
MacroAssemblerX86Shared::extractLaneInt8x16(src, dest, lane,
SimdSign::Unsigned);
}
void MacroAssembler::extractLaneInt16x8(uint32_t lane, FloatRegister src,
Register dest) {
MacroAssemblerX86Shared::extractLaneInt16x8(src, dest, lane,
SimdSign::Signed);
}
void MacroAssembler::unsignedExtractLaneInt16x8(uint32_t lane,
FloatRegister src,
Register dest) {
MacroAssemblerX86Shared::extractLaneInt16x8(src, dest, lane,
SimdSign::Unsigned);
}
void MacroAssembler::extractLaneInt32x4(uint32_t lane, FloatRegister src,
Register dest) {
MacroAssemblerX86Shared::extractLaneInt32x4(src, dest, lane);
}
void MacroAssembler::extractLaneFloat32x4(uint32_t lane, FloatRegister src,
FloatRegister dest) {
MacroAssemblerX86Shared::extractLaneFloat32x4(src, dest, lane);
}
void MacroAssembler::extractLaneFloat64x2(uint32_t lane, FloatRegister src,
FloatRegister dest) {
MacroAssemblerX86Shared::extractLaneFloat64x2(src, dest, lane);
}
// Replace lane value
void MacroAssembler::replaceLaneInt8x16(unsigned lane, FloatRegister lhs,
Register rhs, FloatRegister dest) {
vpinsrb(lane, Operand(rhs), lhs, dest);
}
void MacroAssembler::replaceLaneInt8x16(unsigned lane, Register rhs,
FloatRegister lhsDest) {
vpinsrb(lane, Operand(rhs), lhsDest, lhsDest);
}
void MacroAssembler::replaceLaneInt16x8(unsigned lane, FloatRegister lhs,
Register rhs, FloatRegister dest) {
vpinsrw(lane, Operand(rhs), lhs, dest);
}
void MacroAssembler::replaceLaneInt16x8(unsigned lane, Register rhs,
FloatRegister lhsDest) {
vpinsrw(lane, Operand(rhs), lhsDest, lhsDest);
}
void MacroAssembler::replaceLaneInt32x4(unsigned lane, FloatRegister lhs,
Register rhs, FloatRegister dest) {
vpinsrd(lane, rhs, lhs, dest);
}
void MacroAssembler::replaceLaneInt32x4(unsigned lane, Register rhs,
FloatRegister lhsDest) {
vpinsrd(lane, rhs, lhsDest, lhsDest);
}
void MacroAssembler::replaceLaneFloat32x4(unsigned lane, FloatRegister lhs,
FloatRegister rhs,
FloatRegister dest) {
MacroAssemblerX86Shared::replaceLaneFloat32x4(lane, lhs, rhs, dest);
}
void MacroAssembler::replaceLaneFloat32x4(unsigned lane, FloatRegister rhs,
FloatRegister lhsDest) {
MacroAssemblerX86Shared::replaceLaneFloat32x4(lane, lhsDest, rhs, lhsDest);
}
void MacroAssembler::replaceLaneFloat64x2(unsigned lane, FloatRegister lhs,
FloatRegister rhs,
FloatRegister dest) {
MacroAssemblerX86Shared::replaceLaneFloat64x2(lane, lhs, rhs, dest);
}
void MacroAssembler::replaceLaneFloat64x2(unsigned lane, FloatRegister rhs,
FloatRegister lhsDest) {
MacroAssemblerX86Shared::replaceLaneFloat64x2(lane, lhsDest, rhs, lhsDest);
}
// Shuffle - permute with immediate indices
void MacroAssembler::shuffleInt8x16(const uint8_t lanes[16], FloatRegister rhs,
FloatRegister lhsDest) {
MacroAssemblerX86Shared::shuffleInt8x16(lhsDest, rhs, lhsDest, lanes);
}
void MacroAssembler::shuffleInt8x16(const uint8_t lanes[16], FloatRegister lhs,
FloatRegister rhs, FloatRegister dest) {
MacroAssemblerX86Shared::shuffleInt8x16(lhs, rhs, dest, lanes);
}
void MacroAssembler::blendInt8x16(const uint8_t lanes[16], FloatRegister lhs,
FloatRegister rhs, FloatRegister dest,
FloatRegister temp) {
MacroAssemblerX86Shared::blendInt8x16(lhs, rhs, dest, temp, lanes);
}
void MacroAssembler::blendInt16x8(const uint16_t lanes[8], FloatRegister lhs,
FloatRegister rhs, FloatRegister dest) {
MacroAssemblerX86Shared::blendInt16x8(lhs, rhs, dest, lanes);
}
void MacroAssembler::laneSelectSimd128(FloatRegister mask, FloatRegister lhs,
FloatRegister rhs, FloatRegister dest) {
MacroAssemblerX86Shared::laneSelectSimd128(mask, lhs, rhs, dest);
}
void MacroAssembler::interleaveHighInt16x8(FloatRegister lhs, FloatRegister rhs,
FloatRegister dest) {
vpunpckhwd(rhs, lhs, dest);
}
void MacroAssembler::interleaveHighInt32x4(FloatRegister lhs, FloatRegister rhs,
FloatRegister dest) {
vpunpckhdq(rhs, lhs, dest);
}
void MacroAssembler::interleaveHighInt64x2(FloatRegister lhs, FloatRegister rhs,
FloatRegister dest) {
vpunpckhqdq(rhs, lhs, dest);
}
void MacroAssembler::interleaveHighInt8x16(FloatRegister lhs, FloatRegister rhs,
FloatRegister dest) {
vpunpckhbw(rhs, lhs, dest);
}
void MacroAssembler::interleaveLowInt16x8(FloatRegister lhs, FloatRegister rhs,
FloatRegister dest) {
vpunpcklwd(rhs, lhs, dest);
}
void MacroAssembler::interleaveLowInt32x4(FloatRegister lhs, FloatRegister rhs,
FloatRegister dest) {
vpunpckldq(rhs, lhs, dest);
}
void MacroAssembler::interleaveLowInt64x2(FloatRegister lhs, FloatRegister rhs,
FloatRegister dest) {
vpunpcklqdq(rhs, lhs, dest);
}
void MacroAssembler::interleaveLowInt8x16(FloatRegister lhs, FloatRegister rhs,
FloatRegister dest) {
vpunpcklbw(rhs, lhs, dest);
}
void MacroAssembler::permuteInt8x16(const uint8_t lanes[16], FloatRegister src,
FloatRegister dest) {
src = moveSimd128IntIfNotAVX(src, dest);
vpshufbSimd128(SimdConstant::CreateX16((const int8_t*)lanes), src, dest);
}
void MacroAssembler::permuteLowInt16x8(const uint16_t lanes[4],
FloatRegister src, FloatRegister dest) {
MOZ_ASSERT(lanes[0] < 4 && lanes[1] < 4 && lanes[2] < 4 && lanes[3] < 4);
vpshuflw(ComputeShuffleMask(lanes[0], lanes[1], lanes[2], lanes[3]), src,
dest);
}
void MacroAssembler::permuteHighInt16x8(const uint16_t lanes[4],
FloatRegister src, FloatRegister dest) {
MOZ_ASSERT(lanes[0] < 4 && lanes[1] < 4 && lanes[2] < 4 && lanes[3] < 4);
vpshufhw(ComputeShuffleMask(lanes[0], lanes[1], lanes[2], lanes[3]), src,
dest);