Source code
Revision control
Copy as Markdown
Other Tools
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: set ts=8 sts=2 et sw=2 tw=80:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
#ifndef jit_x86_shared_Assembler_x86_shared_h
#define jit_x86_shared_Assembler_x86_shared_h
#include "mozilla/MathAlgorithms.h"
#include <cstddef>
#include "jit/shared/Assembler-shared.h"
#include "jit/shared/IonAssemblerBuffer.h" // jit::BufferOffset
#if defined(JS_CODEGEN_X86)
# include "jit/x86/BaseAssembler-x86.h"
#elif defined(JS_CODEGEN_X64)
# include "jit/x64/BaseAssembler-x64.h"
#else
# error "Unknown architecture!"
#endif
#include "jit/CompactBuffer.h"
#include "jit/ProcessExecutableMemory.h"
#include "wasm/WasmTypeDecls.h"
namespace js {
namespace jit {
// Do not reference ScratchFloat32Reg_ directly, use ScratchFloat32Scope
// instead.
struct ScratchFloat32Scope : public AutoFloatRegisterScope {
explicit ScratchFloat32Scope(MacroAssembler& masm)
: AutoFloatRegisterScope(masm, ScratchFloat32Reg_) {}
};
// Do not reference ScratchDoubleReg_ directly, use ScratchDoubleScope instead.
struct ScratchDoubleScope : public AutoFloatRegisterScope {
explicit ScratchDoubleScope(MacroAssembler& masm)
: AutoFloatRegisterScope(masm, ScratchDoubleReg_) {}
};
struct ScratchSimd128Scope : public AutoFloatRegisterScope {
explicit ScratchSimd128Scope(MacroAssembler& masm)
: AutoFloatRegisterScope(masm, ScratchSimd128Reg) {}
};
class Operand {
public:
enum Kind {
REG,
MEM_REG_DISP,
FPREG,
MEM_SCALE,
MEM_ADDRESS32,
MEM_SCALE_NOBASE
};
private:
Kind kind_ : 4;
// Used as a Register::Encoding and a FloatRegister::Encoding.
uint32_t base_ : 5;
Scale scale_ : 3;
// We don't use all 8 bits, of course, but GCC complains if the size of
// this field is smaller than the size of Register::Encoding.
Register::Encoding index_ : 8;
int32_t disp_;
public:
explicit Operand(Register reg)
: kind_(REG),
base_(reg.encoding()),
scale_(TimesOne),
index_(Registers::Invalid),
disp_(0) {}
explicit Operand(FloatRegister reg)
: kind_(FPREG),
base_(reg.encoding()),
scale_(TimesOne),
index_(Registers::Invalid),
disp_(0) {}
explicit Operand(const Address& address)
: kind_(MEM_REG_DISP),
base_(address.base.encoding()),
scale_(TimesOne),
index_(Registers::Invalid),
disp_(address.offset) {}
explicit Operand(const BaseIndex& address)
: kind_(MEM_SCALE),
base_(address.base.encoding()),
scale_(address.scale),
index_(address.index.encoding()),
disp_(address.offset) {}
Operand(Register base, Register index, Scale scale, int32_t disp = 0)
: kind_(MEM_SCALE),
base_(base.encoding()),
scale_(scale),
index_(index.encoding()),
disp_(disp) {}
Operand(Register index, Scale scale, int32_t disp)
: kind_(MEM_SCALE_NOBASE),
base_(Registers::Invalid),
scale_(scale),
index_(index.encoding()),
disp_(disp) {}
Operand(Register reg, int32_t disp)
: kind_(MEM_REG_DISP),
base_(reg.encoding()),
scale_(TimesOne),
index_(Registers::Invalid),
disp_(disp) {}
explicit Operand(AbsoluteAddress address)
: kind_(MEM_ADDRESS32),
base_(Registers::Invalid),
scale_(TimesOne),
index_(Registers::Invalid),
disp_(X86Encoding::AddressImmediate(address.addr)) {}
explicit Operand(PatchedAbsoluteAddress address)
: kind_(MEM_ADDRESS32),
base_(Registers::Invalid),
scale_(TimesOne),
index_(Registers::Invalid),
disp_(X86Encoding::AddressImmediate(address.addr)) {}
Address toAddress() const {
MOZ_ASSERT(kind() == MEM_REG_DISP);
return Address(Register::FromCode(base()), disp());
}
BaseIndex toBaseIndex() const {
MOZ_ASSERT(kind() == MEM_SCALE);
return BaseIndex(Register::FromCode(base()), Register::FromCode(index()),
scale(), disp());
}
Kind kind() const { return kind_; }
Register::Encoding reg() const {
MOZ_ASSERT(kind() == REG);
return Register::Encoding(base_);
}
Register::Encoding base() const {
MOZ_ASSERT(kind() == MEM_REG_DISP || kind() == MEM_SCALE);
return Register::Encoding(base_);
}
Register::Encoding index() const {
MOZ_ASSERT(kind() == MEM_SCALE || kind() == MEM_SCALE_NOBASE);
return index_;
}
Scale scale() const {
MOZ_ASSERT(kind() == MEM_SCALE || kind() == MEM_SCALE_NOBASE);
return scale_;
}
FloatRegister::Encoding fpu() const {
MOZ_ASSERT(kind() == FPREG);
return FloatRegister::Encoding(base_);
}
int32_t disp() const {
MOZ_ASSERT(kind() == MEM_REG_DISP || kind() == MEM_SCALE ||
kind() == MEM_SCALE_NOBASE);
return disp_;
}
void* address() const {
MOZ_ASSERT(kind() == MEM_ADDRESS32);
return reinterpret_cast<void*>(disp_);
}
bool containsReg(Register r) const {
switch (kind()) {
case REG:
return r.encoding() == reg();
case MEM_REG_DISP:
return r.encoding() == base();
case MEM_SCALE:
return r.encoding() == base() || r.encoding() == index();
case MEM_SCALE_NOBASE:
return r.encoding() == index();
default:
return false;
}
}
};
class CPUInfo {
public:
// As the SSE's were introduced in order, the presence of a later SSE implies
// the presence of an earlier SSE. For example, SSE4_2 support implies SSE2
// support.
enum SSEVersion {
UnknownSSE = 0,
NoSSE = 1,
SSE = 2,
SSE2 = 3,
SSE3 = 4,
SSSE3 = 5,
SSE4_1 = 6,
SSE4_2 = 7
};
static const int AVX_PRESENT_BIT = 8;
static SSEVersion GetSSEVersion() {
MOZ_ASSERT(FlagsHaveBeenComputed());
MOZ_ASSERT_IF(maxEnabledSSEVersion != UnknownSSE,
maxSSEVersion <= maxEnabledSSEVersion);
return maxSSEVersion;
}
static bool IsAVXPresent() {
MOZ_ASSERT(FlagsHaveBeenComputed());
MOZ_ASSERT_IF(!avxEnabled, !avxPresent);
return avxPresent;
}
static inline uint32_t GetFingerprint() {
return GetSSEVersion() | (IsAVXPresent() ? AVX_PRESENT_BIT : 0);
}
private:
static SSEVersion maxSSEVersion;
static SSEVersion maxEnabledSSEVersion;
static bool avxPresent;
static bool avxEnabled;
static bool popcntPresent;
static bool bmi1Present;
static bool bmi2Present;
static bool lzcntPresent;
static bool fmaPresent;
static bool avx2Present;
static bool f16cPresent;
static void SetMaxEnabledSSEVersion(SSEVersion v) {
if (maxEnabledSSEVersion == UnknownSSE) {
maxEnabledSSEVersion = v;
} else {
maxEnabledSSEVersion = std::min(v, maxEnabledSSEVersion);
}
}
public:
static bool IsSSE2Present() {
#ifdef JS_CODEGEN_X64
return true;
#else
return GetSSEVersion() >= SSE2;
#endif
}
static bool IsSSE3Present() { return GetSSEVersion() >= SSE3; }
static bool IsSSSE3Present() { return GetSSEVersion() >= SSSE3; }
static bool IsSSE41Present() { return GetSSEVersion() >= SSE4_1; }
static bool IsSSE42Present() { return GetSSEVersion() >= SSE4_2; }
static bool IsPOPCNTPresent() { return popcntPresent; }
static bool IsBMI1Present() { return bmi1Present; }
static bool IsBMI2Present() { return bmi2Present; }
static bool IsLZCNTPresent() { return lzcntPresent; }
static bool IsFMAPresent() { return fmaPresent; }
static bool IsAVX2Present() { return avx2Present; }
static bool IsF16CPresent() { return f16cPresent; }
static bool FlagsHaveBeenComputed() { return maxSSEVersion != UnknownSSE; }
static void ComputeFlags();
// The following should be called only before JS_Init (where the flags are
// computed). If several are called, the most restrictive setting is kept.
static void SetSSE3Disabled() {
MOZ_ASSERT(!FlagsHaveBeenComputed());
SetMaxEnabledSSEVersion(SSE2);
avxEnabled = false;
}
static void SetSSSE3Disabled() {
MOZ_ASSERT(!FlagsHaveBeenComputed());
SetMaxEnabledSSEVersion(SSE3);
avxEnabled = false;
}
static void SetSSE41Disabled() {
MOZ_ASSERT(!FlagsHaveBeenComputed());
SetMaxEnabledSSEVersion(SSSE3);
avxEnabled = false;
}
static void SetSSE42Disabled() {
MOZ_ASSERT(!FlagsHaveBeenComputed());
SetMaxEnabledSSEVersion(SSE4_1);
avxEnabled = false;
}
static void SetAVXDisabled() {
MOZ_ASSERT(!FlagsHaveBeenComputed());
avxEnabled = false;
}
static void SetAVXEnabled() {
MOZ_ASSERT(!FlagsHaveBeenComputed());
MOZ_ASSERT(maxEnabledSSEVersion == UnknownSSE,
"Can't enable AVX when SSE has been restricted");
avxEnabled = true;
}
};
class AssemblerX86Shared : public AssemblerShared {
protected:
struct RelativePatch {
int32_t offset;
void* target;
RelocationKind kind;
RelativePatch(int32_t offset, void* target, RelocationKind kind)
: offset(offset), target(target), kind(kind) {}
};
CompactBufferWriter jumpRelocations_;
CompactBufferWriter dataRelocations_;
void writeDataRelocation(ImmGCPtr ptr) {
// Raw GC pointer relocations and Value relocations both end up in
// Assembler::TraceDataRelocations.
if (ptr.value) {
if (gc::IsInsideNursery(ptr.value)) {
embedsNurseryPointers_ = true;
}
dataRelocations_.writeUnsigned(masm.currentOffset());
}
}
protected:
X86Encoding::BaseAssemblerSpecific masm;
using JmpSrc = X86Encoding::JmpSrc;
using JmpDst = X86Encoding::JmpDst;
public:
AssemblerX86Shared() {
if (!HasAVX()) {
masm.disableVEX();
}
}
enum Condition {
Equal = X86Encoding::ConditionE,
NotEqual = X86Encoding::ConditionNE,
Above = X86Encoding::ConditionA,
AboveOrEqual = X86Encoding::ConditionAE,
Below = X86Encoding::ConditionB,
BelowOrEqual = X86Encoding::ConditionBE,
GreaterThan = X86Encoding::ConditionG,
GreaterThanOrEqual = X86Encoding::ConditionGE,
LessThan = X86Encoding::ConditionL,
LessThanOrEqual = X86Encoding::ConditionLE,
Overflow = X86Encoding::ConditionO,
NoOverflow = X86Encoding::ConditionNO,
CarrySet = X86Encoding::ConditionC,
CarryClear = X86Encoding::ConditionNC,
Signed = X86Encoding::ConditionS,
NotSigned = X86Encoding::ConditionNS,
Zero = X86Encoding::ConditionE,
NonZero = X86Encoding::ConditionNE,
Parity = X86Encoding::ConditionP,
NoParity = X86Encoding::ConditionNP
};
enum class SSERoundingMode {
Nearest = int(X86Encoding::SSERoundingMode::RoundToNearest),
Floor = int(X86Encoding::SSERoundingMode::RoundDown),
Ceil = int(X86Encoding::SSERoundingMode::RoundUp),
Trunc = int(X86Encoding::SSERoundingMode::RoundToZero)
};
// If this bit is set, the vucomisd operands have to be inverted.
static const int DoubleConditionBitInvert = 0x10;
// Bit set when a DoubleCondition does not map to a single x86 condition.
// The macro assembler has to special-case these conditions.
static const int DoubleConditionBitSpecial = 0x20;
static const int DoubleConditionBits =
DoubleConditionBitInvert | DoubleConditionBitSpecial;
enum DoubleCondition {
// These conditions will only evaluate to true if the comparison is ordered
// - i.e. neither operand is NaN.
DoubleOrdered = NoParity,
DoubleEqual = Equal | DoubleConditionBitSpecial,
DoubleNotEqual = NotEqual,
DoubleGreaterThan = Above,
DoubleGreaterThanOrEqual = AboveOrEqual,
DoubleLessThan = Above | DoubleConditionBitInvert,
DoubleLessThanOrEqual = AboveOrEqual | DoubleConditionBitInvert,
// If either operand is NaN, these conditions always evaluate to true.
DoubleUnordered = Parity,
DoubleEqualOrUnordered = Equal,
DoubleNotEqualOrUnordered = NotEqual | DoubleConditionBitSpecial,
DoubleGreaterThanOrUnordered = Below | DoubleConditionBitInvert,
DoubleGreaterThanOrEqualOrUnordered =
BelowOrEqual | DoubleConditionBitInvert,
DoubleLessThanOrUnordered = Below,
DoubleLessThanOrEqualOrUnordered = BelowOrEqual
};
enum NaNCond { NaN_HandledByCond, NaN_IsTrue, NaN_IsFalse };
// If the primary condition returned by ConditionFromDoubleCondition doesn't
// handle NaNs properly, return NaN_IsFalse if the comparison should be
// overridden to return false on NaN, NaN_IsTrue if it should be overridden
// to return true on NaN, or NaN_HandledByCond if no secondary check is
// needed.
static inline NaNCond NaNCondFromDoubleCondition(DoubleCondition cond) {
switch (cond) {
case DoubleOrdered:
case DoubleNotEqual:
case DoubleGreaterThan:
case DoubleGreaterThanOrEqual:
case DoubleLessThan:
case DoubleLessThanOrEqual:
case DoubleUnordered:
case DoubleEqualOrUnordered:
case DoubleGreaterThanOrUnordered:
case DoubleGreaterThanOrEqualOrUnordered:
case DoubleLessThanOrUnordered:
case DoubleLessThanOrEqualOrUnordered:
return NaN_HandledByCond;
case DoubleEqual:
return NaN_IsFalse;
case DoubleNotEqualOrUnordered:
return NaN_IsTrue;
}
MOZ_CRASH("Unknown double condition");
}
static void StaticAsserts() {
// DoubleConditionBits should not interfere with x86 condition codes.
static_assert(!((Equal | NotEqual | Above | AboveOrEqual | Below |
BelowOrEqual | Parity | NoParity) &
DoubleConditionBits));
}
static Condition InvertCondition(Condition cond);
static Condition UnsignedCondition(Condition cond);
static Condition ConditionWithoutEqual(Condition cond);
static DoubleCondition InvertCondition(DoubleCondition cond);
// Return the primary condition to test. Some primary conditions may not
// handle NaNs properly and may therefore require a secondary condition.
// Use NaNCondFromDoubleCondition to determine what else is needed.
static inline Condition ConditionFromDoubleCondition(DoubleCondition cond) {
return static_cast<Condition>(cond & ~DoubleConditionBits);
}
static void TraceDataRelocations(JSTracer* trc, JitCode* code,
CompactBufferReader& reader);
void setUnlimitedBuffer() {
// No-op on this platform
}
bool oom() const {
return AssemblerShared::oom() || masm.oom() || jumpRelocations_.oom() ||
dataRelocations_.oom();
}
bool reserve(size_t size) { return masm.reserve(size); }
bool swapBuffer(wasm::Bytes& other) { return masm.swapBuffer(other); }
void setPrinter(Sprinter* sp) { masm.setPrinter(sp); }
Register getStackPointer() const { return StackPointer; }
void executableCopy(void* buffer);
void processCodeLabels(uint8_t* rawCode);
void copyJumpRelocationTable(uint8_t* dest);
void copyDataRelocationTable(uint8_t* dest);
// Size of the instruction stream, in bytes.
size_t size() const { return masm.size(); }
// Size of the jump relocation table, in bytes.
size_t jumpRelocationTableBytes() const { return jumpRelocations_.length(); }
size_t dataRelocationTableBytes() const { return dataRelocations_.length(); }
// Size of the data table, in bytes.
size_t bytesNeeded() const {
return size() + jumpRelocationTableBytes() + dataRelocationTableBytes();
}
public:
void haltingAlign(int alignment) {
MOZ_ASSERT(hasCreator());
masm.haltingAlign(alignment);
}
void nopAlign(int alignment) {
MOZ_ASSERT(hasCreator());
masm.nopAlign(alignment);
}
void writeCodePointer(CodeLabel* label) {
MOZ_ASSERT(hasCreator());
// Use -1 as dummy value. This will be patched after codegen.
masm.jumpTablePointer(-1);
label->patchAt()->bind(masm.size());
}
void cmovCCl(Condition cond, const Operand& src, Register dest) {
X86Encoding::Condition cc = static_cast<X86Encoding::Condition>(cond);
switch (src.kind()) {
case Operand::REG:
masm.cmovCCl_rr(cc, src.reg(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.cmovCCl_mr(cc, src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.cmovCCl_mr(cc, src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void cmovCCl(Condition cond, Register src, Register dest) {
X86Encoding::Condition cc = static_cast<X86Encoding::Condition>(cond);
masm.cmovCCl_rr(cc, src.encoding(), dest.encoding());
}
void cmovzl(const Operand& src, Register dest) {
cmovCCl(Condition::Zero, src, dest);
}
void cmovnzl(const Operand& src, Register dest) {
cmovCCl(Condition::NonZero, src, dest);
}
void movl(Imm32 imm32, Register dest) {
MOZ_ASSERT(hasCreator());
masm.movl_i32r(imm32.value, dest.encoding());
}
void movl(Register src, Register dest) {
MOZ_ASSERT(hasCreator());
masm.movl_rr(src.encoding(), dest.encoding());
}
void movl(const Operand& src, Register dest) {
MOZ_ASSERT(hasCreator());
switch (src.kind()) {
case Operand::REG:
masm.movl_rr(src.reg(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.movl_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.movl_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.movl_mr(src.address(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void movl(Register src, const Operand& dest) {
MOZ_ASSERT(hasCreator());
switch (dest.kind()) {
case Operand::REG:
masm.movl_rr(src.encoding(), dest.reg());
break;
case Operand::MEM_REG_DISP:
masm.movl_rm(src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.movl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
case Operand::MEM_ADDRESS32:
masm.movl_rm(src.encoding(), dest.address());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void movl(Imm32 imm32, const Operand& dest) {
switch (dest.kind()) {
case Operand::REG:
masm.movl_i32r(imm32.value, dest.reg());
break;
case Operand::MEM_REG_DISP:
masm.movl_i32m(imm32.value, dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.movl_i32m(imm32.value, dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
case Operand::MEM_ADDRESS32:
masm.movl_i32m(imm32.value, dest.address());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void xchgl(Register src, Register dest) {
masm.xchgl_rr(src.encoding(), dest.encoding());
}
void vmovapd(FloatRegister src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vmovapd_rr(src.encoding(), dest.encoding());
}
// Eventually vmovapd should be overloaded to support loads and
// stores too.
void vmovapd(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src.kind()) {
case Operand::FPREG:
masm.vmovapd_rr(src.fpu(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovaps(FloatRegister src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vmovaps_rr(src.encoding(), dest.encoding());
}
void vmovaps(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src.kind()) {
case Operand::MEM_REG_DISP:
masm.vmovaps_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vmovaps_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
case Operand::FPREG:
masm.vmovaps_rr(src.fpu(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovaps(FloatRegister src, const Operand& dest) {
MOZ_ASSERT(HasSSE2());
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
masm.vmovaps_rm(src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.vmovaps_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovups(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src.kind()) {
case Operand::MEM_REG_DISP:
masm.vmovups_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vmovups_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovups(FloatRegister src, const Operand& dest) {
MOZ_ASSERT(HasSSE2());
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
masm.vmovups_rm(src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.vmovups_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovsd(const Address& src, FloatRegister dest) {
masm.vmovsd_mr(src.offset, src.base.encoding(), dest.encoding());
}
void vmovsd(const BaseIndex& src, FloatRegister dest) {
masm.vmovsd_mr(src.offset, src.base.encoding(), src.index.encoding(),
src.scale, dest.encoding());
}
void vmovsd(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(hasCreator());
switch (src.kind()) {
case Operand::MEM_REG_DISP:
vmovsd(src.toAddress(), dest);
break;
case Operand::MEM_SCALE:
vmovsd(src.toBaseIndex(), dest);
break;
default:
MOZ_CRASH("Unknown operand for vmovsd");
}
}
void vmovsd(FloatRegister src, const Address& dest) {
masm.vmovsd_rm(src.encoding(), dest.offset, dest.base.encoding());
}
void vmovsd(FloatRegister src, const BaseIndex& dest) {
masm.vmovsd_rm(src.encoding(), dest.offset, dest.base.encoding(),
dest.index.encoding(), dest.scale);
}
// Note special semantics of this - does not clobber high bits of destination.
void vmovsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
masm.vmovsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vmovss(const Address& src, FloatRegister dest) {
masm.vmovss_mr(src.offset, src.base.encoding(), dest.encoding());
}
void vmovss(const BaseIndex& src, FloatRegister dest) {
masm.vmovss_mr(src.offset, src.base.encoding(), src.index.encoding(),
src.scale, dest.encoding());
}
void vmovss(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(hasCreator());
switch (src.kind()) {
case Operand::MEM_REG_DISP:
vmovss(src.toAddress(), dest);
break;
case Operand::MEM_SCALE:
vmovss(src.toBaseIndex(), dest);
break;
default:
MOZ_CRASH("Unknown operand for vmovss");
}
}
void vmovss(FloatRegister src, const Address& dest) {
masm.vmovss_rm(src.encoding(), dest.offset, dest.base.encoding());
}
void vmovss(FloatRegister src, const BaseIndex& dest) {
masm.vmovss_rm(src.encoding(), dest.offset, dest.base.encoding(),
dest.index.encoding(), dest.scale);
}
void vmovss(FloatRegister src, const Operand& dest) {
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
vmovss(src, dest.toAddress());
break;
case Operand::MEM_SCALE:
vmovss(src, dest.toBaseIndex());
break;
default:
MOZ_CRASH("Unknown operand for vmovss");
}
}
// Note special semantics of this - does not clobber high bits of destination.
void vmovss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
masm.vmovss_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vmovdqu(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
MOZ_ASSERT(hasCreator());
switch (src.kind()) {
case Operand::MEM_REG_DISP:
masm.vmovdqu_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vmovdqu_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovdqu(FloatRegister src, const Operand& dest) {
MOZ_ASSERT(HasSSE2());
MOZ_ASSERT(hasCreator());
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
masm.vmovdqu_rm(src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.vmovdqu_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovdqa(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src.kind()) {
case Operand::FPREG:
masm.vmovdqa_rr(src.fpu(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vmovdqa_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vmovdqa_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovdqa(FloatRegister src, const Operand& dest) {
MOZ_ASSERT(HasSSE2());
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
masm.vmovdqa_rm(src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.vmovdqa_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovdqa(FloatRegister src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vmovdqa_rr(src.encoding(), dest.encoding());
}
void vcvtss2sd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vcvtss2sd_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vcvtsd2ss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vcvtsd2ss_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void movzbl(const Operand& src, Register dest) {
switch (src.kind()) {
case Operand::MEM_REG_DISP:
masm.movzbl_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.movzbl_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void movsbl(Register src, Register dest) {
masm.movsbl_rr(src.encoding(), dest.encoding());
}
void movsbl(const Operand& src, Register dest) {
switch (src.kind()) {
case Operand::MEM_REG_DISP:
masm.movsbl_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.movsbl_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void movb(const Operand& src, Register dest) {
switch (src.kind()) {
case Operand::MEM_REG_DISP:
masm.movb_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.movb_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void movb(Imm32 src, Register dest) {
masm.movb_ir(src.value & 255, dest.encoding());
}
void movb(Register src, const Operand& dest) {
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
masm.movb_rm(src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.movb_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void movb(Imm32 src, const Operand& dest) {
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
masm.movb_im(src.value, dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.movb_im(src.value, dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void movzwl(const Operand& src, Register dest) {
switch (src.kind()) {
case Operand::REG:
masm.movzwl_rr(src.reg(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.movzwl_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.movzwl_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void movzwl(Register src, Register dest) {
masm.movzwl_rr(src.encoding(), dest.encoding());
}
void movw(const Operand& src, Register dest) {
masm.prefix_16_for_32();
movl(src, dest);
}
void movw(Imm32 src, Register dest) {
masm.prefix_16_for_32();
movl(src, dest);
}
void movw(Register src, const Operand& dest) {
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
masm.movw_rm(src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.movw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void movw(Imm32 src, const Operand& dest) {
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
masm.movw_im(src.value, dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.movw_im(src.value, dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void movswl(Register src, Register dest) {
masm.movswl_rr(src.encoding(), dest.encoding());
}
void movswl(const Operand& src, Register dest) {
switch (src.kind()) {
case Operand::MEM_REG_DISP:
masm.movswl_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.movswl_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void leal(const Operand& src, Register dest) {
switch (src.kind()) {
case Operand::MEM_REG_DISP:
masm.leal_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.leal_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
case Operand::MEM_SCALE_NOBASE:
masm.leal_mr(src.disp(), src.index(), src.scale(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
protected:
void jSrc(Condition cond, Label* label) {
if (label->bound()) {
// The jump can be immediately encoded to the correct destination.
masm.jCC_i(static_cast<X86Encoding::Condition>(cond),
JmpDst(label->offset()));
} else {
// Thread the jump list through the unpatched jump targets.
JmpSrc j = masm.jCC(static_cast<X86Encoding::Condition>(cond));
JmpSrc prev;
if (label->used()) {
prev = JmpSrc(label->offset());
}
label->use(j.offset());
masm.setNextJump(j, prev);
}
}
void jmpSrc(Label* label) {
if (label->bound()) {
// The jump can be immediately encoded to the correct destination.
masm.jmp_i(JmpDst(label->offset()));
} else {
// Thread the jump list through the unpatched jump targets.
JmpSrc j = masm.jmp();
JmpSrc prev;
if (label->used()) {
prev = JmpSrc(label->offset());
}
label->use(j.offset());
masm.setNextJump(j, prev);
}
}
// Comparison of EAX against the address given by a Label.
JmpSrc cmpSrc(Label* label) {
JmpSrc j = masm.cmp_eax();
if (label->bound()) {
// The jump can be immediately patched to the correct destination.
masm.linkJump(j, JmpDst(label->offset()));
} else {
// Thread the jump list through the unpatched jump targets.
JmpSrc prev;
if (label->used()) {
prev = JmpSrc(label->offset());
}
label->use(j.offset());
masm.setNextJump(j, prev);
}
return j;
}
void bind(Label* label, JmpDst dst) {
if (label->used()) {
bool more;
JmpSrc jmp(label->offset());
do {
JmpSrc next;
more = masm.nextJump(jmp, &next);
masm.linkJump(jmp, dst);
jmp = next;
} while (more);
}
label->bind(dst.offset());
}
public:
void nop() {
MOZ_ASSERT(hasCreator());
masm.nop();
}
void nop(size_t n) {
MOZ_ASSERT(hasCreator());
masm.insert_nop(n);
}
void j(Condition cond, Label* label) {
MOZ_ASSERT(hasCreator());
jSrc(cond, label);
}
void jmp(Label* label) {
MOZ_ASSERT(hasCreator());
jmpSrc(label);
}
void jmp(const Operand& op) {
MOZ_ASSERT(hasCreator());
switch (op.kind()) {
case Operand::MEM_REG_DISP:
masm.jmp_m(op.disp(), op.base());
break;
case Operand::MEM_SCALE:
masm.jmp_m(op.disp(), op.base(), op.index(), op.scale());
break;
case Operand::REG:
masm.jmp_r(op.reg());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void cmpEAX(Label* label) { cmpSrc(label); }
void bind(Label* label) { bind(label, JmpDst(masm.label())); }
void bind(CodeLabel* label) { label->target()->bind(currentOffset()); }
void bind(Label* label, BufferOffset targetOffset) {
bind(label, JmpDst(targetOffset.getOffset()));
}
uint32_t currentOffset() { return masm.label().offset(); }
// Re-routes pending jumps to a new label.
void retarget(Label* label, Label* target) {
if (!label->used()) {
return;
}
bool more;
JmpSrc jmp(label->offset());
do {
JmpSrc next;
more = masm.nextJump(jmp, &next);
if (target->bound()) {
// The jump can be immediately patched to the correct destination.
masm.linkJump(jmp, JmpDst(target->offset()));
} else {
// Thread the jump list through the unpatched jump targets.
JmpSrc prev;
if (target->used()) {
prev = JmpSrc(target->offset());
}
target->use(jmp.offset());
masm.setNextJump(jmp, prev);
}
jmp = JmpSrc(next.offset());
} while (more);
label->reset();
}
static void Bind(uint8_t* raw, const CodeLabel& label) {
if (label.patchAt().bound()) {
intptr_t offset = label.patchAt().offset();
intptr_t target = label.target().offset();
X86Encoding::SetPointer(raw + offset, raw + target);
}
}
BufferOffset ret() {
MOZ_ASSERT(hasCreator());
BufferOffset ret(masm.currentOffset());
masm.ret();
return ret;
}
void retn(Imm32 n) {
MOZ_ASSERT(hasCreator());
// Remove the size of the return address which is included in the frame.
masm.ret_i(n.value - sizeof(void*));
}
CodeOffset call(Label* label) {
JmpSrc j = masm.call();
if (label->bound()) {
masm.linkJump(j, JmpDst(label->offset()));
} else {
JmpSrc prev;
if (label->used()) {
prev = JmpSrc(label->offset());
}
label->use(j.offset());
masm.setNextJump(j, prev);
}
return CodeOffset(masm.currentOffset());
}
CodeOffset call(Register reg) {
masm.call_r(reg.encoding());
return CodeOffset(masm.currentOffset());
}
void call(const Operand& op) {
switch (op.kind()) {
case Operand::REG:
masm.call_r(op.reg());
break;
case Operand::MEM_REG_DISP:
masm.call_m(op.disp(), op.base());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
CodeOffset callWithPatch() { return CodeOffset(masm.call().offset()); }
void patchCall(uint32_t callerOffset, uint32_t calleeOffset) {
unsigned char* code = masm.data();
X86Encoding::SetRel32(code + callerOffset, code + calleeOffset);
}
CodeOffset farJumpWithPatch() { return CodeOffset(masm.jmp().offset()); }
void patchFarJump(CodeOffset farJump, uint32_t targetOffset) {
unsigned char* code = masm.data();
X86Encoding::SetRel32(code + farJump.offset(), code + targetOffset);
}
static void patchFarJump(uint8_t* farJump, uint8_t* target) {
MOZ_RELEASE_ASSERT(mozilla::Abs(target - farJump) <=
(intptr_t)jit::MaxCodeBytesPerProcess);
X86Encoding::SetRel32(farJump, target);
}
// This is for patching during code generation, not after.
void patchAddl(CodeOffset offset, int32_t n) {
unsigned char* code = masm.data();
X86Encoding::SetInt32(code + offset.offset(), n);
}
static void patchFiveByteNopToCall(uint8_t* callsite, uint8_t* target) {
X86Encoding::BaseAssembler::patchFiveByteNopToCall(callsite, target);
}
static void patchCallToFiveByteNop(uint8_t* callsite) {
X86Encoding::BaseAssembler::patchCallToFiveByteNop(callsite);
}
void breakpoint() { masm.int3(); }
CodeOffset ud2() {
MOZ_ASSERT(hasCreator());
CodeOffset off(masm.currentOffset());
masm.ud2();
return off;
}
static bool HasSSE2() { return CPUInfo::IsSSE2Present(); }
static bool HasSSE3() { return CPUInfo::IsSSE3Present(); }
static bool HasSSSE3() { return CPUInfo::IsSSSE3Present(); }
static bool HasSSE41() { return CPUInfo::IsSSE41Present(); }
static bool HasSSE42() { return CPUInfo::IsSSE42Present(); }
static bool HasPOPCNT() { return CPUInfo::IsPOPCNTPresent(); }
static bool HasBMI1() { return CPUInfo::IsBMI1Present(); }
static bool HasBMI2() { return CPUInfo::IsBMI2Present(); }
static bool HasLZCNT() { return CPUInfo::IsLZCNTPresent(); }
static bool HasF16C() { return CPUInfo::IsF16CPresent(); }
static bool SupportsFloatingPoint() { return CPUInfo::IsSSE2Present(); }
static bool SupportsUnalignedAccesses() { return true; }
static bool SupportsFastUnalignedFPAccesses() { return true; }
static bool SupportsWasmSimd() { return CPUInfo::IsSSE41Present(); }
static bool SupportsFloat64To16() { return false; }
static bool SupportsFloat32To16() { return CPUInfo::IsF16CPresent(); }
static bool HasAVX() { return CPUInfo::IsAVXPresent(); }
static bool HasAVX2() { return CPUInfo::IsAVX2Present(); }
static bool HasFMA() { return CPUInfo::IsFMAPresent(); }
static bool HasRoundInstruction(RoundingMode mode) {
switch (mode) {
case RoundingMode::Up:
case RoundingMode::Down:
case RoundingMode::NearestTiesToEven:
case RoundingMode::TowardsZero:
return CPUInfo::IsSSE41Present();
}
MOZ_CRASH("unexpected mode");
}
void cmpl(Register rhs, Register lhs) {
masm.cmpl_rr(rhs.encoding(), lhs.encoding());
}
void cmpl(const Operand& rhs, Register lhs) {
switch (rhs.kind()) {
case Operand::REG:
masm.cmpl_rr(rhs.reg(), lhs.encoding());
break;
case Operand::MEM_REG_DISP:
masm.cmpl_mr(rhs.disp(), rhs.base(), lhs.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.cmpl_mr(rhs.address(), lhs.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void cmpl(Register rhs, const Operand& lhs) {
switch (lhs.kind()) {
case Operand::REG:
masm.cmpl_rr(rhs.encoding(), lhs.reg());
break;
case Operand::MEM_REG_DISP:
masm.cmpl_rm(rhs.encoding(), lhs.disp(), lhs.base());
break;
case Operand::MEM_SCALE:
masm.cmpl_rm(rhs.encoding(), lhs.disp(), lhs.base(), lhs.index(),
lhs.scale());
break;
case Operand::MEM_ADDRESS32:
masm.cmpl_rm(rhs.encoding(), lhs.address());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void cmpl(Imm32 rhs, Register lhs) {
masm.cmpl_ir(rhs.value, lhs.encoding());
}
void cmpl(Imm32 rhs, const Operand& lhs) {
switch (lhs.kind()) {
case Operand::REG:
masm.cmpl_ir(rhs.value, lhs.reg());
break;
case Operand::MEM_REG_DISP:
masm.cmpl_im(rhs.value, lhs.disp(), lhs.base());
break;
case Operand::MEM_SCALE:
masm.cmpl_im(rhs.value, lhs.disp(), lhs.base(), lhs.index(),
lhs.scale());
break;
case Operand::MEM_ADDRESS32:
masm.cmpl_im(rhs.value, lhs.address());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void cmpw(Register rhs, Register lhs) {
masm.cmpw_rr(rhs.encoding(), lhs.encoding());
}
void cmpw(Imm32 rhs, const Operand& lhs) {
switch (lhs.kind()) {
case Operand::REG:
masm.cmpw_ir(rhs.value, lhs.reg());
break;
case Operand::MEM_REG_DISP:
masm.cmpw_im(rhs.value, lhs.disp(), lhs.base());
break;
case Operand::MEM_SCALE:
masm.cmpw_im(rhs.value, lhs.disp(), lhs.base(), lhs.index(),
lhs.scale());
break;
case Operand::MEM_ADDRESS32:
masm.cmpw_im(rhs.value, lhs.address());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void cmpb(Register rhs, const Operand& lhs) {
switch (lhs.kind()) {
case Operand::REG:
masm.cmpb_rr(rhs.encoding(), lhs.reg());
break;
case Operand::MEM_REG_DISP:
masm.cmpb_rm(rhs.encoding(), lhs.disp(), lhs.base());
break;
case Operand::MEM_SCALE:
masm.cmpb_rm(rhs.encoding(), lhs.disp(), lhs.base(), lhs.index(),
lhs.scale());
break;
case Operand::MEM_ADDRESS32:
masm.cmpb_rm(rhs.encoding(), lhs.address());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void cmpb(Imm32 rhs, const Operand& lhs) {
switch (lhs.kind()) {
case Operand::REG:
masm.cmpb_ir(rhs.value, lhs.reg());
break;
case Operand::MEM_REG_DISP:
masm.cmpb_im(rhs.value, lhs.disp(), lhs.base());
break;
case Operand::MEM_SCALE:
masm.cmpb_im(rhs.value, lhs.disp(), lhs.base(), lhs.index(),
lhs.scale());
break;
case Operand::MEM_ADDRESS32:
masm.cmpb_im(rhs.value, lhs.address());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void setCC(Condition cond, Register r) {
masm.setCC_r(static_cast<X86Encoding::Condition>(cond), r.encoding());
}
void testb(Register rhs, Register lhs) {
MOZ_ASSERT(
AllocatableGeneralRegisterSet(Registers::SingleByteRegs).has(rhs));
MOZ_ASSERT(
AllocatableGeneralRegisterSet(Registers::SingleByteRegs).has(lhs));
masm.testb_rr(rhs.encoding(), lhs.encoding());
}
void testw(Register rhs, Register lhs) {
masm.testw_rr(lhs.encoding(), rhs.encoding());
}
void testl(Register rhs, Register lhs) {
masm.testl_rr(lhs.encoding(), rhs.encoding());
}
void testl(Imm32 rhs, Register lhs) {
masm.testl_ir(rhs.value, lhs.encoding());
}
void testl(Imm32 rhs, const Operand& lhs) {
switch (lhs.kind()) {
case Operand::REG:
masm.testl_ir(rhs.value, lhs.reg());
break;
case Operand::MEM_REG_DISP:
masm.testl_i32m(rhs.value, lhs.disp(), lhs.base());
break;
case Operand::MEM_ADDRESS32:
masm.testl_i32m(rhs.value, lhs.address());
break;
default:
MOZ_CRASH("unexpected operand kind");
break;
}
}
void addl(Imm32 imm, Register dest) {
masm.addl_ir(imm.value, dest.encoding());
}
CodeOffset addlWithPatch(Imm32 imm, Register dest) {
masm.addl_i32r(imm.value, dest.encoding());
return CodeOffset(masm.currentOffset());
}
void addl(Imm32 imm, const Operand& op) {
switch (op.kind()) {
case Operand::REG:
masm.addl_ir(imm.value, op.reg());
break;
case Operand::MEM_REG_DISP:
masm.addl_im(imm.value, op.disp(), op.base());
break;
case Operand::MEM_ADDRESS32:
masm.addl_im(imm.value, op.address());
break;
case Operand::MEM_SCALE:
masm.addl_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void addw(Imm32 imm, const Operand& op) {
switch (op.kind()) {
case Operand::REG:
masm.addw_ir(imm.value, op.reg());
break;
case Operand::MEM_REG_DISP:
masm.addw_im(imm.value, op.disp(), op.base());
break;
case Operand::MEM_ADDRESS32:
masm.addw_im(imm.value, op.address());
break;
case Operand::MEM_SCALE:
masm.addw_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void subl(Imm32 imm, Register dest) {
masm.subl_ir(imm.value, dest.encoding());
}
size_t subl(Imm32 imm, const Operand& op) {
switch (op.kind()) {
case Operand::REG:
return masm.subl_ir(imm.value, op.reg());
case Operand::MEM_REG_DISP:
return masm.subl_im(imm.value, op.disp(), op.base());
case Operand::MEM_SCALE:
return masm.subl_im(imm.value, op.disp(), op.base(), op.index(),
op.scale());
default:
MOZ_CRASH("unexpected operand kind");
}
}
void subw(Imm32 imm, const Operand& op) {
switch (op.kind()) {
case Operand::REG:
masm.subw_ir(imm.value, op.reg());
break;
case Operand::MEM_REG_DISP:
masm.subw_im(imm.value, op.disp(), op.base());
break;
case Operand::MEM_SCALE:
masm.subw_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void addl(Register src, Register dest) {
masm.addl_rr(src.encoding(), dest.encoding());
}
void addl(Register src, const Operand& dest) {
switch (dest.kind()) {
case Operand::REG:
masm.addl_rr(src.encoding(), dest.reg());
break;
case Operand::MEM_REG_DISP:
masm.addl_rm(src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.addl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void addw(Register src, const Operand& dest) {
switch (dest.kind()) {
case Operand::REG:
masm.addw_rr(src.encoding(), dest.reg());
break;
case Operand::MEM_REG_DISP:
masm.addw_rm(src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.addw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void sbbl(Register src, Register dest) {
masm.sbbl_rr(src.encoding(), dest.encoding());
}
void subl(Register src, Register dest) {
masm.subl_rr(src.encoding(), dest.encoding());
}
void subl(const Operand& src, Register dest) {
switch (src.kind()) {
case Operand::REG:
masm.subl_rr(src.reg(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.subl_mr(src.disp(), src.base(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void subl(Register src, const Operand& dest) {
switch (dest.kind()) {
case Operand::REG:
masm.subl_rr(src.encoding(), dest.reg());
break;
case Operand::MEM_REG_DISP:
masm.subl_rm(src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.subl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void subw(Register src, const Operand& dest) {
switch (dest.kind()) {
case Operand::REG:
masm.subw_rr(src.encoding(), dest.reg());
break;
case Operand::MEM_REG_DISP:
masm.subw_rm(src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.subw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void orl(Register reg, Register dest) {
masm.orl_rr(reg.encoding(), dest.encoding());
}
void orl(Register src, const Operand& dest) {
switch (dest.kind()) {
case Operand::REG:
masm.orl_rr(src.encoding(), dest.reg());
break;
case Operand::MEM_REG_DISP:
masm.orl_rm(src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.orl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void orw(Register src, const Operand& dest) {
switch (dest.kind()) {
case Operand::REG:
masm.orw_rr(src.encoding(), dest.reg());
break;
case Operand::MEM_REG_DISP:
masm.orw_rm(src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.orw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void orl(Imm32 imm, Register reg) { masm.orl_ir(imm.value, reg.encoding()); }
void orl(Imm32 imm, const Operand& op) {
switch (op.kind()) {
case Operand::REG:
masm.orl_ir(imm.value, op.reg());
break;
case Operand::MEM_REG_DISP:
masm.orl_im(imm.value, op.disp(), op.base());
break;
case Operand::MEM_SCALE:
masm.orl_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void orw(Imm32 imm, const Operand& op) {
switch (op.kind()) {
case Operand::REG:
masm.orw_ir(imm.value, op.reg());
break;
case Operand::MEM_REG_DISP:
masm.orw_im(imm.value, op.disp(), op.base());
break;
case Operand::MEM_SCALE:
masm.orw_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void xorl(Register src, Register dest) {
masm.xorl_rr(src.encoding(), dest.encoding());
}
void xorl(Register src, const Operand& dest) {
switch (dest.kind()) {
case Operand::REG:
masm.xorl_rr(src.encoding(), dest.reg());
break;
case Operand::MEM_REG_DISP:
masm.xorl_rm(src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.xorl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void xorw(Register src, const Operand& dest) {
switch (dest.kind()) {
case Operand::REG:
masm.xorw_rr(src.encoding(), dest.reg());
break;
case Operand::MEM_REG_DISP:
masm.xorw_rm(src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.xorw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void xorl(Imm32 imm, Register reg) {
masm.xorl_ir(imm.value, reg.encoding());
}
void xorl(Imm32 imm, const Operand& op) {
switch (op.kind()) {
case Operand::REG:
masm.xorl_ir(imm.value, op.reg());
break;
case Operand::MEM_REG_DISP:
masm.xorl_im(imm.value, op.disp(), op.base());
break;
case Operand::MEM_SCALE:
masm.xorl_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void xorw(Imm32 imm, const Operand& op) {
switch (op.kind()) {
case Operand::REG:
masm.xorw_ir(imm.value, op.reg());
break;
case Operand::MEM_REG_DISP:
masm.xorw_im(imm.value, op.disp(), op.base());
break;
case Operand::MEM_SCALE:
masm.xorw_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void andl(Register src, Register dest) {
masm.andl_rr(src.encoding(), dest.encoding());
}
void andl(Register src, const Operand& dest) {
switch (dest.kind()) {
case Operand::REG:
masm.andl_rr(src.encoding(), dest.reg());
break;
case Operand::MEM_REG_DISP:
masm.andl_rm(src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.andl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void andw(Register src, const Operand& dest) {
switch (dest.kind()) {
case Operand::REG:
masm.andw_rr(src.encoding(), dest.reg());
break;
case Operand::MEM_REG_DISP:
masm.andw_rm(src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.andw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void andl(Imm32 imm, Register dest) {
masm.andl_ir(imm.value, dest.encoding());
}
void andl(Imm32 imm, const Operand& op) {
switch (op.kind()) {
case Operand::REG:
masm.andl_ir(imm.value, op.reg());
break;
case Operand::MEM_REG_DISP:
masm.andl_im(imm.value, op.disp(), op.base());
break;
case Operand::MEM_SCALE:
masm.andl_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void andw(Imm32 imm, const Operand& op) {
switch (op.kind()) {
case Operand::REG:
masm.andw_ir(imm.value, op.reg());
break;
case Operand::MEM_REG_DISP:
masm.andw_im(imm.value, op.disp(), op.base());
break;
case Operand::MEM_SCALE:
masm.andw_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void addl(const Operand& src, Register dest) {
switch (src.kind()) {
case Operand::REG:
masm.addl_rr(src.reg(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.addl_mr(src.disp(), src.base(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void andnl(Register src1, Register src2, Register dest) {
MOZ_ASSERT(HasBMI1());
masm.andnl_rrr(src1.encoding(), src2.encoding(), dest.encoding());
}
void orl(const Operand& src, Register dest) {
switch (src.kind()) {
case Operand::REG:
masm.orl_rr(src.reg(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.orl_mr(src.disp(), src.base(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void xorl(const Operand& src, Register dest) {
switch (src.kind()) {
case Operand::REG:
masm.xorl_rr(src.reg(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.xorl_mr(src.disp(), src.base(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void andl(const Operand& src, Register dest) {
switch (src.kind()) {
case Operand::REG:
masm.andl_rr(src.reg(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.andl_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.andl_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void bsrl(const Register& src, const Register& dest) {
masm.bsrl_rr(src.encoding(), dest.encoding());
}
void bsfl(const Register& src, const Register& dest) {
masm.bsfl_rr(src.encoding(), dest.encoding());
}
void bswapl(Register reg) { masm.bswapl_r(reg.encoding()); }
void lzcntl(const Register& src, const Register& dest) {
masm.lzcntl_rr(src.encoding(), dest.encoding());
}
void tzcntl(const Register& src, const Register& dest) {
masm.tzcntl_rr(src.encoding(), dest.encoding());
}
void popcntl(const Register& src, const Register& dest) {
masm.popcntl_rr(src.encoding(), dest.encoding());
}
void imull(Register multiplier) {
// Consumes eax as the other argument
// and clobbers edx, as result is in edx:eax
masm.imull_r(multiplier.encoding());
}
void umull(Register multiplier) { masm.mull_r(multiplier.encoding()); }
void imull(Imm32 imm, Register dest) {
masm.imull_ir(imm.value, dest.encoding(), dest.encoding());
}
void imull(Register src, Register dest) {
masm.imull_rr(src.encoding(), dest.encoding());
}
void imull(Imm32 imm, Register src, Register dest) {
masm.imull_ir(imm.value, src.encoding(), dest.encoding());
}
void imull(const Operand& src, Register dest) {
switch (src.kind()) {
case Operand::REG:
masm.imull_rr(src.reg(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.imull_mr(src.disp(), src.base(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void negl(const Operand& src) {
switch (src.kind()) {
case Operand::REG:
masm.negl_r(src.reg());
break;
case Operand::MEM_REG_DISP:
masm.negl_m(src.disp(), src.base());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void negl(Register reg) { masm.negl_r(reg.encoding()); }
void notl(const Operand& src) {
switch (src.kind()) {
case Operand::REG:
masm.notl_r(src.reg());
break;
case Operand::MEM_REG_DISP:
masm.notl_m(src.disp(), src.base());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void notl(Register reg) { masm.notl_r(reg.encoding()); }
void shrl(const Imm32 imm, Register dest) {
masm.shrl_ir(imm.value, dest.encoding());
}
void shll(const Imm32 imm, Register dest) {
masm.shll_ir(imm.value, dest.encoding());
}
void sarl(const Imm32 imm, Register dest) {
masm.sarl_ir(imm.value, dest.encoding());
}
void shrl_cl(Register dest) { masm.shrl_CLr(dest.encoding()); }
void shll_cl(Register dest) { masm.shll_CLr(dest.encoding()); }
void sarl_cl(Register dest) { masm.sarl_CLr(dest.encoding()); }
void shrdl_cl(Register src, Register dest) {
masm.shrdl_CLr(src.encoding(), dest.encoding());
}
void shldl_cl(Register src, Register dest) {
masm.shldl_CLr(src.encoding(), dest.encoding());
}
void sarxl(Register src, Register shift, Register dest) {
MOZ_ASSERT(HasBMI2());
masm.sarxl_rrr(src.encoding(), shift.encoding(), dest.encoding());
}
void shlxl(Register src, Register shift, Register dest) {
MOZ_ASSERT(HasBMI2());
masm.shlxl_rrr(src.encoding(), shift.encoding(), dest.encoding());
}
void shrxl(Register src, Register shift, Register dest) {
MOZ_ASSERT(HasBMI2());
masm.shrxl_rrr(src.encoding(), shift.encoding(), dest.encoding());
}
void roll(const Imm32 imm, Register dest) {
masm.roll_ir(imm.value, dest.encoding());
}
void roll_cl(Register dest) { masm.roll_CLr(dest.encoding()); }
void rolw(const Imm32 imm, Register dest) {
masm.rolw_ir(imm.value, dest.encoding());
}
void rorl(const Imm32 imm, Register dest) {
masm.rorl_ir(imm.value, dest.encoding());
}
void rorl_cl(Register dest) { masm.rorl_CLr(dest.encoding()); }
void incl(const Operand& op) {
switch (op.kind()) {
case Operand::MEM_REG_DISP:
masm.incl_m32(op.disp(), op.base());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void lock_incl(const Operand& op) {
masm.prefix_lock();
incl(op);
}
void decl(const Operand& op) {
switch (op.kind()) {
case Operand::MEM_REG_DISP:
masm.decl_m32(op.disp(), op.base());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void lock_decl(const Operand& op) {
masm.prefix_lock();
decl(op);
}
void addb(Imm32 imm, const Operand& op) {
switch (op.kind()) {
case Operand::MEM_REG_DISP:
masm.addb_im(imm.value, op.disp(), op.base());
break;
case Operand::MEM_SCALE:
masm.addb_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
break;
}
}
void addb(Register src, const Operand& op) {
switch (op.kind()) {
case Operand::MEM_REG_DISP:
masm.addb_rm(src.encoding(), op.disp(), op.base());
break;
case Operand::MEM_SCALE:
masm.addb_rm(src.encoding(), op.disp(), op.base(), op.index(),
op.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
break;
}
}
void subb(Imm32 imm, const Operand& op) {
switch (op.kind()) {
case Operand::MEM_REG_DISP:
masm.subb_im(imm.value, op.disp(), op.base());
break;
case Operand::MEM_SCALE:
masm.subb_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
break;
}
}
void subb(Register src, const Operand& op) {
switch (op.kind()) {
case Operand::MEM_REG_DISP:
masm.subb_rm(src.encoding(), op.disp(), op.base());
break;
case Operand::MEM_SCALE:
masm.subb_rm(src.encoding(), op.disp(), op.base(), op.index(),
op.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
break;
}
}
void andb(Imm32 imm, const Operand& op) {
switch (op.kind()) {
case Operand::MEM_REG_DISP:
masm.andb_im(imm.value, op.disp(), op.base());
break;
case Operand::MEM_SCALE:
masm.andb_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
break;
}
}
void andb(Register src, const Operand& op) {
switch (op.kind()) {
case Operand::MEM_REG_DISP:
masm.andb_rm(src.encoding(), op.disp(), op.base());
break;
case Operand::MEM_SCALE:
masm.andb_rm(src.encoding(), op.disp(), op.base(), op.index(),
op.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
break;
}
}
void orb(Imm32 imm, const Operand& op) {
switch (op.kind()) {
case Operand::MEM_REG_DISP:
masm.orb_im(imm.value, op.disp(), op.base());
break;
case Operand::MEM_SCALE:
masm.orb_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
break;
}
}
void orb(Register src, const Operand& op) {
switch (op.kind()) {
case Operand::MEM_REG_DISP:
masm.orb_rm(src.encoding(), op.disp(), op.base());
break;
case Operand::MEM_SCALE:
masm.orb_rm(src.encoding(), op.disp(), op.base(), op.index(),
op.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
break;
}
}
void xorb(Imm32 imm, const Operand& op) {
switch (op.kind()) {
case Operand::MEM_REG_DISP:
masm.xorb_im(imm.value, op.disp(), op.base());
break;
case Operand::MEM_SCALE:
masm.xorb_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
break;
}
}
void xorb(Register src, const Operand& op) {
switch (op.kind()) {
case Operand::MEM_REG_DISP:
masm.xorb_rm(src.encoding(), op.disp(), op.base());
break;
case Operand::MEM_SCALE:
masm.xorb_rm(src.encoding(), op.disp(), op.base(), op.index(),
op.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
break;
}
}
template <typename T>
void lock_addb(T src, const Operand& op) {
masm.prefix_lock();
addb(src, op);
}
template <typename T>
void lock_subb(T src, const Operand& op) {
masm.prefix_lock();
subb(src, op);
}
template <typename T>
void lock_andb(T src, const Operand& op) {
masm.prefix_lock();
andb(src, op);
}
template <typename T>
void lock_orb(T src, const Operand& op) {
masm.prefix_lock();
orb(src, op);
}
template <typename T>
void lock_xorb(T src, const Operand& op) {
masm.prefix_lock();
xorb(src, op);
}
template <typename T>
void lock_addw(T src, const Operand& op) {
masm.prefix_lock();
addw(src, op);
}
template <typename T>
void lock_subw(T src, const Operand& op) {
masm.prefix_lock();
subw(src, op);
}
template <typename T>
void lock_andw(T src, const Operand& op) {
masm.prefix_lock();
andw(src, op);
}
template <typename T>
void lock_orw(T src, const Operand& op) {
masm.prefix_lock();
orw(src, op);
}
template <typename T>
void lock_xorw(T src, const Operand& op) {
masm.prefix_lock();
xorw(src, op);
}
// Note, lock_addl(imm, op) is used for a memory barrier on non-SSE2 systems,
// among other things. Do not optimize, replace by XADDL, or similar.
template <typename T>
void lock_addl(T src, const Operand& op) {
masm.prefix_lock();
addl(src, op);
}
template <typename T>
void lock_subl(T src, const Operand& op) {
masm.prefix_lock();
subl(src, op);
}
template <typename T>
void lock_andl(T src, const Operand& op) {
masm.prefix_lock();
andl(src, op);
}
template <typename T>
void lock_orl(T src, const Operand& op) {
masm.prefix_lock();
orl(src, op);
}
template <typename T>
void lock_xorl(T src, const Operand& op) {
masm.prefix_lock();
xorl(src, op);
}
void lock_cmpxchgb(Register src, const Operand& mem) {
masm.prefix_lock();
switch (mem.kind()) {
case Operand::MEM_REG_DISP:
masm.cmpxchgb(src.encoding(), mem.disp(), mem.base());
break;
case Operand::MEM_SCALE:
masm.cmpxchgb(src.encoding(), mem.disp(), mem.base(), mem.index(),
mem.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void lock_cmpxchgw(Register src, const Operand& mem) {
masm.prefix_lock();
switch (mem.kind()) {
case Operand::MEM_REG_DISP:
masm.cmpxchgw(src.encoding(), mem.disp(), mem.base());
break;
case Operand::MEM_SCALE:
masm.cmpxchgw(src.encoding(), mem.disp(), mem.base(), mem.index(),
mem.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void lock_cmpxchgl(Register src, const Operand& mem) {
masm.prefix_lock();
switch (mem.kind()) {
case Operand::MEM_REG_DISP:
masm.cmpxchgl(src.encoding(), mem.disp(), mem.base());
break;
case Operand::MEM_SCALE:
masm.cmpxchgl(src.encoding(), mem.disp(), mem.base(), mem.index(),
mem.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void lock_cmpxchg8b(Register srcHi, Register srcLo, Register newHi,
Register newLo, const Operand& mem) {
masm.prefix_lock();
switch (mem.kind()) {
case Operand::MEM_REG_DISP:
masm.cmpxchg8b(srcHi.encoding(), srcLo.encoding(), newHi.encoding(),
newLo.encoding(), mem.disp(), mem.base());
break;
case Operand::MEM_SCALE:
masm.cmpxchg8b(srcHi.encoding(), srcLo.encoding(), newHi.encoding(),
newLo.encoding(), mem.disp(), mem.base(), mem.index(),
mem.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void xchgb(Register src, const Operand& mem) {
switch (mem.kind()) {
case Operand::MEM_REG_DISP:
masm.xchgb_rm(src.encoding(), mem.disp(), mem.base());
break;
case Operand::MEM_SCALE:
masm.xchgb_rm(src.encoding(), mem.disp(), mem.base(), mem.index(),
mem.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void xchgw(Register src, const Operand& mem) {
switch (mem.kind()) {
case Operand::MEM_REG_DISP:
masm.xchgw_rm(src.encoding(), mem.disp(), mem.base());
break;
case Operand::MEM_SCALE:
masm.xchgw_rm(src.encoding(), mem.disp(), mem.base(), mem.index(),
mem.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void xchgl(Register src, const Operand& mem) {
switch (mem.kind()) {
case Operand::MEM_REG_DISP:
masm.xchgl_rm(src.encoding(), mem.disp(), mem.base());
break;
case Operand::MEM_SCALE:
masm.xchgl_rm(src.encoding(), mem.disp(), mem.base(), mem.index(),
mem.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void lock_xaddb(Register srcdest, const Operand& mem) {
switch (mem.kind()) {
case Operand::MEM_REG_DISP:
masm.lock_xaddb_rm(srcdest.encoding(), mem.disp(), mem.base());
break;
case Operand::MEM_SCALE:
masm.lock_xaddb_rm(srcdest.encoding(), mem.disp(), mem.base(),
mem.index(), mem.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void lock_xaddw(Register srcdest, const Operand& mem) {
masm.prefix_16_for_32();
lock_xaddl(srcdest, mem);
}
void lock_xaddl(Register srcdest, const Operand& mem) {
switch (mem.kind()) {
case Operand::MEM_REG_DISP:
masm.lock_xaddl_rm(srcdest.encoding(), mem.disp(), mem.base());
break;
case Operand::MEM_SCALE:
masm.lock_xaddl_rm(srcdest.encoding(), mem.disp(), mem.base(),
mem.index(), mem.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void push(const Imm32 imm) { masm.push_i(imm.value); }
void push(const Operand& src) {
MOZ_ASSERT(hasCreator());
switch (src.kind()) {
case Operand::REG:
masm.push_r(src.reg());
break;
case Operand::MEM_REG_DISP:
masm.push_m(src.disp(), src.base());
break;
case Operand::MEM_SCALE:
masm.push_m(src.disp(), src.base(), src.index(), src.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void push(Register src) {
MOZ_ASSERT(hasCreator());
masm.push_r(src.encoding());
}
void push(const Address& src) {
masm.push_m(src.offset, src.base.encoding());
}
void pop(const Operand& src) {
MOZ_ASSERT(hasCreator());
switch (src.kind()) {
case Operand::REG:
masm.pop_r(src.reg());
break;
case Operand::MEM_REG_DISP:
masm.pop_m(src.disp(), src.base());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void pop(Register src) {
MOZ_ASSERT(hasCreator());
masm.pop_r(src.encoding());
}
void pop(const Address& src) { masm.pop_m(src.offset, src.base.encoding()); }
void pushFlags() { masm.push_flags(); }
void popFlags() { masm.pop_flags(); }
#ifdef JS_CODEGEN_X86
void pushAllRegs() { masm.pusha(); }
void popAllRegs() { masm.popa(); }
#endif
// Zero-extend byte to 32-bit integer.
void movzbl(Register src, Register dest) {
masm.movzbl_rr(src.encoding(), dest.encoding());
}
void cdq() { masm.cdq(); }
void idiv(Register divisor) { masm.idivl_r(divisor.encoding()); }
void udiv(Register divisor) { masm.divl_r(divisor.encoding()); }
void vpblendw(uint32_t mask, FloatRegister src1, FloatRegister src0,
FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
masm.vpblendw_irr(mask, src1.encoding(), src0.encoding(), dest.encoding());
}
void vpblendvb(FloatRegister mask, FloatRegister src1, FloatRegister src0,
FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
masm.vpblendvb_rr(mask.encoding(), src1.encoding(), src0.encoding(),
dest.encoding());
}
void vpinsrb(unsigned lane, const Operand& src1, FloatRegister src0,
FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
switch (src1.kind()) {
case Operand::REG:
masm.vpinsrb_irr(lane, src1.reg(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpinsrb_imr(lane, src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vpinsrb_imr(lane, src1.disp(), src1.base(), src1.index(),
src1.scale(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpinsrw(unsigned lane, const Operand& src1, FloatRegister src0,
FloatRegister dest) {
switch (src1.kind()) {
case Operand::REG:
masm.vpinsrw_irr(lane, src1.reg(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpinsrw_imr(lane, src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vpinsrw_imr(lane, src1.disp(), src1.base(), src1.index(),
src1.scale(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpinsrd(unsigned lane, Register src1, FloatRegister src0,
FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
masm.vpinsrd_irr(lane, src1.encoding(), src0.encoding(), dest.encoding());
}
void vpextrb(unsigned lane, FloatRegister src, const Operand& dest) {
MOZ_ASSERT(HasSSE41());
switch (dest.kind()) {
case Operand::REG:
masm.vpextrb_irr(lane, src.encoding(), dest.reg());
break;
case Operand::MEM_REG_DISP:
masm.vpextrb_irm(lane, src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.vpextrb_irm(lane, src.encoding(), dest.disp(), dest.base(),
dest.index(), dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpextrw(unsigned lane, FloatRegister src, const Operand& dest) {
MOZ_ASSERT(HasSSE41());
switch (dest.kind()) {
case Operand::REG:
masm.vpextrw_irr(lane, src.encoding(), dest.reg());
break;
case Operand::MEM_REG_DISP:
masm.vpextrw_irm(lane, src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.vpextrw_irm(lane, src.encoding(), dest.disp(), dest.base(),
dest.index(), dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpextrd(unsigned lane, FloatRegister src, Register dest) {
MOZ_ASSERT(HasSSE41());
masm.vpextrd_irr(lane, src.encoding(), dest.encoding());
}
void vpsrldq(Imm32 shift, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vpsrldq_ir(shift.value, src0.encoding(), dest.encoding());
}
void vpslldq(Imm32 shift, FloatRegister src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vpslldq_ir(shift.value, src.encoding(), dest.encoding());
}
void vpsllq(Imm32 shift, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vpsllq_ir(shift.value, src0.encoding(), dest.encoding());
}
void vpsllq(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vpsllq_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vpsrlq(Imm32 shift, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vpsrlq_ir(shift.value, src0.encoding(), dest.encoding());
}
void vpsrlq(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vpsrlq_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vpslld(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vpslld_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vpslld(Imm32 count, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vpslld_ir(count.value, src0.encoding(), dest.encoding());
}
void vpsrad(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vpsrad_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vpsrad(Imm32 count, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vpsrad_ir(count.value, src0.encoding(), dest.encoding());
}
void vpsrld(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vpsrld_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vpsrld(Imm32 count, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vpsrld_ir(count.value, src0.encoding(), dest.encoding());
}
void vpsllw(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vpsllw_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vpsllw(Imm32 count, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vpsllw_ir(count.value, src0.encoding(), dest.encoding());
}
void vpsraw(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vpsraw_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vpsraw(Imm32 count, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vpsraw_ir(count.value, src0.encoding(), dest.encoding());
}
void vpsrlw(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vpsrlw_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vpsrlw(Imm32 count, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vpsrlw_ir(count.value, src0.encoding(), dest.encoding());
}
void vcvtsi2sd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::REG:
masm.vcvtsi2sd_rr(src1.reg(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vcvtsi2sd_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vcvtsi2sd_mr(src1.disp(), src1.base(), src1.index(), src1.scale(),
src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vcvttsd2si(FloatRegister src, Register dest) {
MOZ_ASSERT(HasSSE2());
masm.vcvttsd2si_rr(src.encoding(), dest.encoding());
}
void vcvttss2si(FloatRegister src, Register dest) {
MOZ_ASSERT(HasSSE2());
masm.vcvttss2si_rr(src.encoding(), dest.encoding());
}
void vcvtsi2ss(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::REG:
masm.vcvtsi2ss_rr(src1.reg(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vcvtsi2ss_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vcvtsi2ss_mr(src1.disp(), src1.base(), src1.index(), src1.scale(),
src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vcvtsi2ss(Register src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vcvtsi2ss_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vcvtsi2sd(Register src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vcvtsi2sd_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vcvttps2dq(FloatRegister src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vcvttps2dq_rr(src.encoding(), dest.encoding());
}
void vcvttpd2dq(FloatRegister src, FloatRegister dest) {
masm.vcvttpd2dq_rr(src.encoding(), dest.encoding());
}
void vcvtdq2ps(FloatRegister src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vcvtdq2ps_rr(src.encoding(), dest.encoding());
}
void vcvtdq2pd(FloatRegister src, FloatRegister dest) {
masm.vcvtdq2pd_rr(src.encoding(), dest.encoding());
}
void vcvtps2pd(FloatRegister src, FloatRegister dest) {
masm.vcvtps2pd_rr(src.encoding(), dest.encoding());
}
void vcvtpd2ps(FloatRegister src, FloatRegister dest) {
masm.vcvtpd2ps_rr(src.encoding(), dest.encoding());
}
void vcvtph2ps(FloatRegister src, FloatRegister dest) {
MOZ_ASSERT(HasF16C());
masm.vcvtph2ps_rr(src.encoding(), dest.encoding());
}
void vcvtps2ph(FloatRegister src, FloatRegister dest) {
MOZ_ASSERT(HasF16C());
masm.vcvtps2ph_rr(src.encoding(), dest.encoding());
}
void vmovmskpd(FloatRegister src, Register dest) {
MOZ_ASSERT(HasSSE2());
masm.vmovmskpd_rr(src.encoding(), dest.encoding());
}
void vmovmskps(FloatRegister src, Register dest) {
MOZ_ASSERT(HasSSE2());
masm.vmovmskps_rr(src.encoding(), dest.encoding());
}
void vpmovmskb(FloatRegister src, Register dest) {
MOZ_ASSERT(HasSSE2());
masm.vpmovmskb_rr(src.encoding(), dest.encoding());
}
void vptest(FloatRegister rhs, FloatRegister lhs) {
MOZ_ASSERT(HasSSE41());
masm.vptest_rr(rhs.encoding(), lhs.encoding());
}
void vucomisd(FloatRegister rhs, FloatRegister lhs) {
MOZ_ASSERT(HasSSE2());
masm.vucomisd_rr(rhs.encoding(), lhs.encoding());
}
void vucomiss(FloatRegister rhs, FloatRegister lhs) {
MOZ_ASSERT(HasSSE2());
masm.vucomiss_rr(rhs.encoding(), lhs.encoding());
}
void vpcmpeqb(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (rhs.kind()) {
case Operand::FPREG:
masm.vpcmpeqb_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpcmpeqb_mr(rhs.disp(), rhs.base(), lhs.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpcmpeqb_mr(rhs.address(), lhs.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpcmpgtb(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (rhs.kind()) {
case Operand::FPREG:
masm.vpcmpgtb_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpcmpgtb_mr(rhs.disp(), rhs.base(), lhs.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpcmpgtb_mr(rhs.address(), lhs.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpcmpeqw(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (rhs.kind()) {
case Operand::FPREG:
masm.vpcmpeqw_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpcmpeqw_mr(rhs.disp(), rhs.base(), lhs.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpcmpeqw_mr(rhs.address(), lhs.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpcmpgtw(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (rhs.kind()) {
case Operand::FPREG:
masm.vpcmpgtw_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpcmpgtw_mr(rhs.disp(), rhs.base(), lhs.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpcmpgtw_mr(rhs.address(), lhs.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpcmpeqd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (rhs.kind()) {
case Operand::FPREG:
masm.vpcmpeqd_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpcmpeqd_mr(rhs.disp(), rhs.base(), lhs.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpcmpeqd_mr(rhs.address(), lhs.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpcmpgtd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (rhs.kind()) {
case Operand::FPREG:
masm.vpcmpgtd_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpcmpgtd_mr(rhs.disp(), rhs.base(), lhs.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpcmpgtd_mr(rhs.address(), lhs.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpcmpgtq(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
MOZ_ASSERT(HasSSE42());
switch (rhs.kind()) {
case Operand::FPREG:
masm.vpcmpgtq_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpcmpeqq(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
switch (rhs.kind()) {
case Operand::FPREG:
masm.vpcmpeqq_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpcmpeqq_mr(rhs.disp(), rhs.base(), lhs.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpcmpeqq_mr(rhs.address(), lhs.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vcmpps(uint8_t order, Operand rhs, FloatRegister lhs,
FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (rhs.kind()) {
case Operand::FPREG:
masm.vcmpps_rr(order, rhs.fpu(), lhs.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vcmpps_mr(order, rhs.disp(), rhs.base(), lhs.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vcmpps_mr(order, rhs.address(), lhs.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vcmpeqps(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
vcmpps(X86Encoding::ConditionCmp_EQ, rhs, lhs, dest);
}
void vcmpltps(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
vcmpps(X86Encoding::ConditionCmp_LT, rhs, lhs, dest);
}
void vcmpleps(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
vcmpps(X86Encoding::ConditionCmp_LE, rhs, lhs, dest);
}
void vcmpunordps(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
vcmpps(X86Encoding::ConditionCmp_UNORD, rhs, lhs, dest);
}
void vcmpneqps(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
vcmpps(X86Encoding::ConditionCmp_NEQ, rhs, lhs, dest);
}
void vcmpordps(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
vcmpps(X86Encoding::ConditionCmp_ORD, rhs, lhs, dest);
}
void vcmppd(uint8_t order, Operand rhs, FloatRegister lhs,
FloatRegister dest) {
switch (rhs.kind()) {
case Operand::FPREG:
masm.vcmppd_rr(order, rhs.fpu(), lhs.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("NYI");
}
}
void vcmpeqpd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
vcmppd(X86Encoding::ConditionCmp_EQ, rhs, lhs, dest);
}
void vcmpltpd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
vcmppd(X86Encoding::ConditionCmp_LT, rhs, lhs, dest);
}
void vcmplepd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
vcmppd(X86Encoding::ConditionCmp_LE, rhs, lhs, dest);
}
void vcmpneqpd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
vcmppd(X86Encoding::ConditionCmp_NEQ, rhs, lhs, dest);
}
void vcmpordpd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
vcmppd(X86Encoding::ConditionCmp_ORD, rhs, lhs, dest);
}
void vcmpunordpd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
vcmppd(X86Encoding::ConditionCmp_UNORD, rhs, lhs, dest);
}
void vrcpps(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src.kind()) {
case Operand::FPREG:
masm.vrcpps_rr(src.fpu(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vrcpps_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vrcpps_mr(src.address(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vsqrtps(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src.kind()) {
case Operand::FPREG:
masm.vsqrtps_rr(src.fpu(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vsqrtps_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vsqrtps_mr(src.address(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vrsqrtps(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src.kind()) {
case Operand::FPREG:
masm.vrsqrtps_rr(src.fpu(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vrsqrtps_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vrsqrtps_mr(src.address(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vsqrtpd(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src.kind()) {
case Operand::FPREG:
masm.vsqrtpd_rr(src.fpu(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovd(Register src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vmovd_rr(src.encoding(), dest.encoding());
}
void vmovd(FloatRegister src, Register dest) {
MOZ_ASSERT(HasSSE2());
masm.vmovd_rr(src.encoding(), dest.encoding());
}
void vmovd(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src.kind()) {
case Operand::MEM_REG_DISP:
masm.vmovd_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vmovd_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovd(FloatRegister src, const Operand& dest) {
MOZ_ASSERT(HasSSE2());
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
masm.vmovd_rm(src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.vmovd_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
case Operand::MEM_ADDRESS32:
masm.vmovq_rm(src.encoding(), dest.address());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovq(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src.kind()) {
case Operand::MEM_REG_DISP:
masm.vmovq_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vmovq_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vmovq_mr(src.address(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovq(FloatRegister src, const Operand& dest) {
MOZ_ASSERT(HasSSE2());
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
masm.vmovq_rm(src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.vmovq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpmaddubsw(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSSE3());
masm.vpmaddubsw_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vpaddb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpaddb_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpaddb_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpaddb_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpsubb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpsubb_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpsubb_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpsubb_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpaddsb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpaddsb_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpaddsb_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpaddsb_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpaddusb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpaddusb_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpaddusb_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpaddusb_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpsubsb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpsubsb_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpsubsb_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpsubsb_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpsubusb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpsubusb_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpsubusb_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpsubusb_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpaddw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpaddw_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpaddw_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpaddw_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpsubw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpsubw_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpsubw_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpsubw_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpaddsw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpaddsw_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpaddsw_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpaddsw_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpaddusw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpaddusw_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpaddusw_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpaddusw_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpsubsw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpsubsw_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpsubsw_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpsubsw_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpsubusw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpsubusw_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpsubusw_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpsubusw_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpaddd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpaddd_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpaddd_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpaddd_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpsubd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpsubd_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpsubd_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpsubd_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpmuldq(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
masm.vpmuldq_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vpmuludq(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vpmuludq_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vpmuludq(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpmuludq_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpmuludq_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpmullw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpmullw_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpmullw_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpmulhw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpmulhw_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpmulhw_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpmulhuw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpmulhuw_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpmulhuw_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpmulhrsw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpmulhrsw_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpmulhrsw_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpmulld(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpmulld_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpmulld_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpmulld_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpmaddwd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpmaddwd_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpaddq(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpaddq_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpsubq(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpsubq_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vaddps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vaddps_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vaddps_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vaddps_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vsubps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vsubps_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vsubps_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vsubps_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmulps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vmulps_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vmulps_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vmulps_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vdivps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vdivps_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vdivps_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vdivps_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmaxps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vmaxps_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vmaxps_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vmaxps_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vminps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vminps_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vminps_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vminps_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vminpd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vminpd_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmaxpd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vmaxpd_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vaddpd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vaddpd_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vsubpd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vsubpd_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmulpd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vmulpd_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vdivpd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vdivpd_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpavgb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpavgb_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpavgw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpavgw_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpminsb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpminsb_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpminub(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpminub_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpmaxsb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpmaxsb_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpmaxub(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpmaxub_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpminsw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpminsw_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpminuw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpminuw_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpmaxsw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpmaxsw_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpmaxuw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpmaxuw_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpminsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpminsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpminud(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpminud_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpmaxsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpmaxsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpmaxud(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpmaxud_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpacksswb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpacksswb_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpackuswb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpackuswb_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpackssdw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpackssdw_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpackusdw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpackusdw_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpabsb(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE3());
switch (src.kind()) {
case Operand::FPREG:
masm.vpabsb_rr(src.fpu(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpabsw(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE3());
switch (src.kind()) {
case Operand::FPREG:
masm.vpabsw_rr(src.fpu(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpabsd(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE3());
switch (src.kind()) {
case Operand::FPREG:
masm.vpabsd_rr(src.fpu(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpmovsxbw(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
switch (src.kind()) {
case Operand::FPREG:
masm.vpmovsxbw_rr(src.fpu(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpmovsxbw_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vpmovsxbw_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpmovzxbw(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
switch (src.kind()) {
case Operand::FPREG:
masm.vpmovzxbw_rr(src.fpu(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpmovzxbw_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vpmovzxbw_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpmovzxbd(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
switch (src.kind()) {
case Operand::FPREG:
masm.vpmovzxbd_rr(src.fpu(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpmovzxbq(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
switch (src.kind()) {
case Operand::FPREG:
masm.vpmovzxbq_rr(src.fpu(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpmovsxwd(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
switch (src.kind()) {
case Operand::FPREG:
masm.vpmovsxwd_rr(src.fpu(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpmovsxwd_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vpmovsxwd_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpmovzxwd(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
switch (src.kind()) {
case Operand::FPREG:
masm.vpmovzxwd_rr(src.fpu(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpmovzxwd_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vpmovzxwd_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpmovzxwq(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
switch (src.kind()) {
case Operand::FPREG:
masm.vpmovzxwq_rr(src.fpu(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpmovsxdq(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
switch (src.kind()) {
case Operand::FPREG:
masm.vpmovsxdq_rr(src.fpu(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpmovsxdq_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vpmovsxdq_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpmovzxdq(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
switch (src.kind()) {
case Operand::FPREG:
masm.vpmovzxdq_rr(src.fpu(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpmovzxdq_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vpmovzxdq_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vphaddd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
switch (src1.kind()) {
case Operand::FPREG:
masm.vphaddd_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpalignr(const Operand& src1, FloatRegister src0, FloatRegister dest,
uint8_t shift) {
MOZ_ASSERT(HasSSE3());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpalignr_irr(shift, src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpunpcklbw(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
MOZ_ASSERT(src0.size() == 16);
MOZ_ASSERT(src1.size() == 16);
MOZ_ASSERT(dest.size() == 16);
masm.vpunpcklbw_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vpunpckhbw(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
MOZ_ASSERT(src0.size() == 16);
MOZ_ASSERT(src1.size() == 16);
MOZ_ASSERT(dest.size() == 16);
masm.vpunpckhbw_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vpunpcklbw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpunpcklbw_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpunpckldq(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
MOZ_ASSERT(src0.size() == 16);
MOZ_ASSERT(src1.size() == 16);
MOZ_ASSERT(dest.size() == 16);
masm.vpunpckldq_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vpunpckldq(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
MOZ_ASSERT(src0.size() == 16);
MOZ_ASSERT(dest.size() == 16);
switch (src1.kind()) {
case Operand::MEM_REG_DISP:
masm.vpunpckldq_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpunpckldq_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpunpcklqdq(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
MOZ_ASSERT(src0.size() == 16);
MOZ_ASSERT(src1.size() == 16);
MOZ_ASSERT(dest.size() == 16);
masm.vpunpcklqdq_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vpunpcklqdq(const Operand& src1, FloatRegister src0,
FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
MOZ_ASSERT(src0.size() == 16);
MOZ_ASSERT(dest.size() == 16);
switch (src1.kind()) {
case Operand::MEM_REG_DISP:
masm.vpunpcklqdq_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpunpcklqdq_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpunpckhdq(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
MOZ_ASSERT(src0.size() == 16);
MOZ_ASSERT(src1.size() == 16);
MOZ_ASSERT(dest.size() == 16);
masm.vpunpckhdq_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vpunpckhqdq(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
MOZ_ASSERT(src0.size() == 16);
MOZ_ASSERT(src1.size() == 16);
MOZ_ASSERT(dest.size() == 16);
masm.vpunpckhqdq_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vpunpcklwd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
MOZ_ASSERT(src0.size() == 16);
MOZ_ASSERT(src1.size() == 16);
MOZ_ASSERT(dest.size() == 16);
masm.vpunpcklwd_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vpunpckhwd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
MOZ_ASSERT(src0.size() == 16);
MOZ_ASSERT(src1.size() == 16);
MOZ_ASSERT(dest.size() == 16);
masm.vpunpckhwd_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vandps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vandps_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vandps_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vandps_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vandnps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
// Negates bits of dest and then applies AND
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vandnps_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vandnps_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vandnps_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vorps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vorps_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vorps_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vorps_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vxorps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vxorps_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vxorps_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vxorps_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vandpd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vandpd_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpand(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vpand_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vpand(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpand_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpand_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpand_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpor(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vpor_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vpor(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpor_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpor_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpor_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpxor(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vpxor_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vpxor(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpxor_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpxor_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpxor_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpandn(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vpandn_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vpandn(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpandn_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpandn_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpandn_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpshufd(uint32_t mask, FloatRegister src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vpshufd_irr(mask, src.encoding(), dest.encoding());
}
void vpshufd(uint32_t mask, const Operand& src1, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpshufd_irr(mask, src1.fpu(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vpshufd_imr(mask, src1.disp(), src1.base(), dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vpshufd_imr(mask, src1.address(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpshuflw(uint32_t mask, FloatRegister src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vpshuflw_irr(mask, src.encoding(), dest.encoding());
}
void vpshufhw(uint32_t mask, FloatRegister src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vpshufhw_irr(mask, src.encoding(), dest.encoding());
}
void vpshufb(FloatRegister mask, FloatRegister src, FloatRegister dest) {
MOZ_ASSERT(HasSSSE3());
masm.vpshufb_rr(mask.encoding(), src.encoding(), dest.encoding());
}
void vmovddup(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE3());
switch (src.kind()) {
case Operand::FPREG:
masm.vmovddup_rr(src.fpu(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vmovddup_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vmovddup_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovhlps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vmovhlps_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vmovlhps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vmovlhps_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vunpcklps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vunpcklps_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vunpcklps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vunpcklps_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vunpcklps_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vunpcklps_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vunpckhps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vunpckhps_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vunpckhps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vunpckhps_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vunpckhps_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vunpckhps_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vshufps(uint32_t mask, FloatRegister src1, FloatRegister src0,
FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vshufps_irr(mask, src1.encoding(), src0.encoding(), dest.encoding());
}
void vshufps(uint32_t mask, const Operand& src1, FloatRegister src0,
FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vshufps_irr(mask, src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vshufps_imr(mask, src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vshufps_imr(mask, src1.address(), src0.encoding(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vshufpd(uint32_t mask, FloatRegister src1, FloatRegister src0,
FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vshufpd_irr(mask, src1.encoding(), src0.encoding(), dest.encoding());
}
void vaddsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vaddsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vaddss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vaddss_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vaddsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vaddsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vaddsd_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vaddsd_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vaddss(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vaddss_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vaddss_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.vaddss_mr(src1.address(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vsubsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vsubsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vsubss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vsubss_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vsubsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vsubsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vsubsd_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vsubss(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vsubss_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vsubss_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmulsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vmulsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vmulsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vmulsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vmulsd_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmulss(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vmulss_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vmulss_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmulss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vmulss_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vdivsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vdivsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vdivss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vdivss_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vdivsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vdivsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vdivsd_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vdivss(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vdivss_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vdivss_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vxorpd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vxorpd_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vxorps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vxorps_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vorpd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vorpd_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vorps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vorps_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vandpd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vandpd_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vandps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vandps_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vsqrtsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vsqrtsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vsqrtss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vsqrtss_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vroundps(SSERoundingMode mode, const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
switch (src.kind()) {
case Operand::FPREG:
masm.vroundps_irr((X86Encoding::SSERoundingMode)mode, src.fpu(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vroundpd(SSERoundingMode mode, const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
switch (src.kind()) {
case Operand::FPREG:
masm.vroundpd_irr((X86Encoding::SSERoundingMode)mode, src.fpu(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
static X86Encoding::RoundingMode ToX86RoundingMode(RoundingMode mode) {
switch (mode) {
case RoundingMode::Up:
return X86Encoding::RoundUp;
case RoundingMode::Down:
return X86Encoding::RoundDown;
case RoundingMode::NearestTiesToEven:
return X86Encoding::RoundToNearest;
case RoundingMode::TowardsZero:
return X86Encoding::RoundToZero;
}
MOZ_CRASH("unexpected mode");
}
void vroundsd(X86Encoding::RoundingMode mode, FloatRegister src,
FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
masm.vroundsd_irr(mode, src.encoding(), dest.encoding());
}
void vroundss(X86Encoding::RoundingMode mode, FloatRegister src,
FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
masm.vroundss_irr(mode, src.encoding(), dest.encoding());
}
unsigned vinsertpsMask(unsigned sourceLane, unsigned destLane,
unsigned zeroMask = 0) {
// Note that the sourceLane bits are ignored in the case of a source
// memory operand, and the source is the given 32-bits memory location.
MOZ_ASSERT(zeroMask < 16);
unsigned ret = zeroMask;
ret |= destLane << 4;
ret |= sourceLane << 6;
MOZ_ASSERT(ret < 256);
return ret;
}
void vinsertps(uint32_t mask, FloatRegister src1, FloatRegister src0,
FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
masm.vinsertps_irr(mask, src1.encoding(), src0.encoding(), dest.encoding());
}
void vinsertps(uint32_t mask, const Operand& src1, FloatRegister src0,
FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
switch (src1.kind()) {
case Operand::FPREG:
masm.vinsertps_irr(mask, src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vinsertps_imr(mask, src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vinsertps_imr(mask, src1.disp(), src1.base(), src1.index(),
src1.scale(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovlps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
switch (src1.kind()) {
case Operand::MEM_REG_DISP:
masm.vmovlps_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vmovlps_mr(src1.disp(), src1.base(), src1.index(), src1.scale(),
src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovlps(FloatRegister src, const Operand& dest) {
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
masm.vmovlps_rm(src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.vmovlps_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovhps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
switch (src1.kind()) {
case Operand::MEM_REG_DISP:
masm.vmovhps_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vmovhps_mr(src1.disp(), src1.base(), src1.index(), src1.scale(),
src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovhps(FloatRegister src, const Operand& dest) {
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
masm.vmovhps_rm(src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.vmovhps_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vextractps(unsigned lane, FloatRegister src, const Operand& dest) {
MOZ_ASSERT(HasSSE41());
MOZ_ASSERT(lane < 4);
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
masm.vextractps_rm(lane, src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.vextractps_rm(lane, src.encoding(), dest.disp(), dest.base(),
dest.index(), dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
unsigned blendpsMask(bool x, bool y, bool z, bool w) {
return (x << 0) | (y << 1) | (z << 2) | (w << 3);
}
void vblendps(unsigned mask, FloatRegister src1, FloatRegister src0,
FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
masm.vblendps_irr(mask, src1.encoding(), src0.encoding(), dest.encoding());
}
void vblendps(unsigned mask, const Operand& src1, FloatRegister src0,
FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
switch (src1.kind()) {
case Operand::FPREG:
masm.vblendps_irr(mask, src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vblendps_imr(mask, src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vblendvps(FloatRegister mask, FloatRegister src1, FloatRegister src0,
FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
masm.vblendvps_rr(mask.encoding(), src1.encoding(), src0.encoding(),
dest.encoding());
}
void vblendvps(FloatRegister mask, const Operand& src1, FloatRegister src0,
FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
switch (src1.kind()) {
case Operand::FPREG:
masm.vblendvps_rr(mask.encoding(), src1.fpu(), src0.encoding(),
dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vblendvps_mr(mask.encoding(), src1.disp(), src1.base(),
src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vblendvpd(FloatRegister mask, FloatRegister src1, FloatRegister src0,
FloatRegister dest) {
MOZ_ASSERT(HasSSE41());
masm.vblendvpd_rr(mask.encoding(), src1.encoding(), src0.encoding(),
dest.encoding());
}
void vmovsldup(FloatRegister src, FloatRegister dest) {
MOZ_ASSERT(HasSSE3());
masm.vmovsldup_rr(src.encoding(), dest.encoding());
}
void vmovsldup(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE3());
switch (src.kind()) {
case Operand::FPREG:
masm.vmovsldup_rr(src.fpu(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vmovsldup_mr(src.disp(), src.base(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovshdup(FloatRegister src, FloatRegister dest) {
MOZ_ASSERT(HasSSE3());
masm.vmovshdup_rr(src.encoding(), dest.encoding());
}
void vmovshdup(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE3());
switch (src.kind()) {
case Operand::FPREG:
masm.vmovshdup_rr(src.fpu(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vmovshdup_mr(src.disp(), src.base(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vminsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vminsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vminsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vminsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vminsd_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vminss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vminss_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vmaxsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vmaxsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vmaxsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src1.kind()) {
case Operand::FPREG:
masm.vmaxsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vmaxsd_mr(src1.disp(), src1.base(), src0.encoding(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmaxss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vmaxss_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void fisttp(const Operand& dest) {
MOZ_ASSERT(HasSSE3());
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
masm.fisttp_m(dest.disp(), dest.base());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void fistp(const Operand& dest) {
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
masm.fistp_m(dest.disp(), dest.base());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void fnstcw(const Operand& dest) {
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
masm.fnstcw_m(dest.disp(), dest.base());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void fldcw(const Operand& dest) {
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
masm.fldcw_m(dest.disp(), dest.base());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void fnstsw(const Operand& dest) {
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
masm.fnstsw_m(dest.disp(), dest.base());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void fld(const Operand& dest) {
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
masm.fld_m(dest.disp(), dest.base());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void fld32(const Operand& dest) {
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
masm.fld32_m(dest.disp(), dest.base());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void fstp(const Operand& src) {
switch (src.kind()) {
case Operand::MEM_REG_DISP:
masm.fstp_m(src.disp(), src.base());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void fstp32(const Operand& src) {
switch (src.kind()) {
case Operand::MEM_REG_DISP:
masm.fstp32_m(src.disp(), src.base());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vbroadcastb(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasAVX2());
switch (src.kind()) {
case Operand::FPREG:
masm.vbroadcastb_rr(src.fpu(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vbroadcastb_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vbroadcastb_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vbroadcastw(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasAVX2());
switch (src.kind()) {
case Operand::FPREG:
masm.vbroadcastw_rr(src.fpu(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vbroadcastw_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vbroadcastw_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vbroadcastd(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasAVX2());
switch (src.kind()) {
case Operand::FPREG:
masm.vbroadcastd_rr(src.fpu(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vbroadcastd_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vbroadcastd_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vbroadcastq(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasAVX2());
switch (src.kind()) {
case Operand::FPREG:
masm.vbroadcastq_rr(src.fpu(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vbroadcastq_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vbroadcastq_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vbroadcastss(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasAVX2());
switch (src.kind()) {
case Operand::FPREG:
masm.vbroadcastss_rr(src.fpu(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vbroadcastss_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vbroadcastss_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vpsignd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSSE3());
switch (src1.kind()) {
case Operand::FPREG:
masm.vpsignd_rr(src1.fpu(), src0.encoding(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vfmadd231ps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasFMA());
masm.vfmadd231ps_rrr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vfnmadd231ps(FloatRegister src1, FloatRegister src0,
FloatRegister dest) {
MOZ_ASSERT(HasFMA());
masm.vfnmadd231ps_rrr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vfmadd231pd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasFMA());
masm.vfmadd231pd_rrr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vfnmadd231pd(FloatRegister src1, FloatRegister src0,
FloatRegister dest) {
MOZ_ASSERT(HasFMA());
masm.vfnmadd231pd_rrr(src1.encoding(), src0.encoding(), dest.encoding());
}
void flushBuffer() {}
// Patching.
static size_t PatchWrite_NearCallSize() { return 5; }
static uintptr_t GetPointer(uint8_t* instPtr) {
uint8_t* ptr = instPtr - sizeof(uintptr_t);
return mozilla::LittleEndian::readUintptr(ptr);
}
// Write a relative call at the start location |dataLabel|.
// Note that this DOES NOT patch data that comes before |label|.
static void PatchWrite_NearCall(CodeLocationLabel startLabel,
CodeLocationLabel target) {
uint8_t* start = startLabel.raw();
*start = 0xE8; // <CALL> rel32
ptrdiff_t offset = target - startLabel - PatchWrite_NearCallSize();
MOZ_ASSERT(int32_t(offset) == offset);
mozilla::LittleEndian::writeInt32(start + 1, offset); // CALL <rel32>
}
static void PatchWrite_Imm32(CodeLocationLabel dataLabel, Imm32 toWrite) {
// dataLabel is a code location which targets the end of an instruction
// which has a 32 bits immediate. Thus writting a value requires shifting
// back to the address of the 32 bits immediate within the instruction.
uint8_t* ptr = dataLabel.raw();
mozilla::LittleEndian::writeInt32(ptr - sizeof(int32_t), toWrite.value);
}
static void PatchDataWithValueCheck(CodeLocationLabel data,
PatchedImmPtr newData,
PatchedImmPtr expectedData) {
// The pointer given is a pointer to *after* the data.
uint8_t* ptr = data.raw() - sizeof(uintptr_t);
MOZ_ASSERT(mozilla::LittleEndian::readUintptr(ptr) ==
uintptr_t(expectedData.value));
mozilla::LittleEndian::writeUintptr(ptr, uintptr_t(newData.value));
}
static void PatchDataWithValueCheck(CodeLocationLabel data, ImmPtr newData,
ImmPtr expectedData) {
PatchDataWithValueCheck(data, PatchedImmPtr(newData.value),
PatchedImmPtr(expectedData.value));
}
static uint32_t NopSize() { return 1; }
static uint8_t* NextInstruction(uint8_t* cur, uint32_t* count) {
MOZ_CRASH("nextInstruction NYI on x86");
}
// Toggle a jmp or cmp emitted by toggledJump().
static void ToggleToJmp(CodeLocationLabel inst) {
uint8_t* ptr = (uint8_t*)inst.raw();
MOZ_ASSERT(*ptr == 0x3D); // <CMP> eax, imm32
*ptr = 0xE9; // <JMP> rel32
}
static void ToggleToCmp(CodeLocationLabel inst) {
uint8_t* ptr = (uint8_t*)inst.raw();
MOZ_ASSERT(*ptr == 0xE9); // <JMP> rel32
*ptr = 0x3D; // <CMP> eax, imm32
}
static void ToggleCall(CodeLocationLabel inst, bool enabled) {
uint8_t* ptr = (uint8_t*)inst.raw();
MOZ_ASSERT(*ptr == 0x3D || // <CMP> eax, imm32
*ptr == 0xE8); // <CALL> rel32
*ptr = enabled ? 0xE8 : 0x3D;
}
MOZ_COLD void verifyHeapAccessDisassembly(
uint32_t begin, uint32_t end, const Disassembler::HeapAccess& heapAccess);
};
} // namespace jit
} // namespace js
#endif /* jit_x86_shared_Assembler_x86_shared_h */