Source code

Revision control

Copy as Markdown

Other Tools

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: set ts=8 sts=2 et sw=2 tw=80:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef jit_x86_shared_Assembler_x86_shared_h
#define jit_x86_shared_Assembler_x86_shared_h
#include "mozilla/MathAlgorithms.h"
#include <cstddef>
#include "jit/shared/Assembler-shared.h"
#include "jit/shared/IonAssemblerBuffer.h" // jit::BufferOffset
#if defined(JS_CODEGEN_X86)
# include "jit/x86/BaseAssembler-x86.h"
#elif defined(JS_CODEGEN_X64)
# include "jit/x64/BaseAssembler-x64.h"
#else
# error "Unknown architecture!"
#endif
#include "jit/CompactBuffer.h"
#include "jit/ProcessExecutableMemory.h"
#include "wasm/WasmTypeDecls.h"
namespace js {
namespace jit {
// Do not reference ScratchFloat32Reg_ directly, use ScratchFloat32Scope
// instead.
struct ScratchFloat32Scope : public AutoFloatRegisterScope {
explicit ScratchFloat32Scope(MacroAssembler& masm)
: AutoFloatRegisterScope(masm, ScratchFloat32Reg_) {}
};
// Do not reference ScratchDoubleReg_ directly, use ScratchDoubleScope instead.
struct ScratchDoubleScope : public AutoFloatRegisterScope {
explicit ScratchDoubleScope(MacroAssembler& masm)
: AutoFloatRegisterScope(masm, ScratchDoubleReg_) {}
};
struct ScratchSimd128Scope : public AutoFloatRegisterScope {
explicit ScratchSimd128Scope(MacroAssembler& masm)
: AutoFloatRegisterScope(masm, ScratchSimd128Reg) {}
};
class Operand {
public:
enum Kind { REG, MEM_REG_DISP, FPREG, MEM_SCALE, MEM_ADDRESS32 };
private:
Kind kind_ : 4;
// Used as a Register::Encoding and a FloatRegister::Encoding.
uint32_t base_ : 5;
Scale scale_ : 3;
// We don't use all 8 bits, of course, but GCC complains if the size of
// this field is smaller than the size of Register::Encoding.
Register::Encoding index_ : 8;
int32_t disp_;
public:
explicit Operand(Register reg)
: kind_(REG),
base_(reg.encoding()),
scale_(TimesOne),
index_(Registers::Invalid),
disp_(0) {}
explicit Operand(FloatRegister reg)
: kind_(FPREG),
base_(reg.encoding()),
scale_(TimesOne),
index_(Registers::Invalid),
disp_(0) {}
explicit Operand(const Address& address)
: kind_(MEM_REG_DISP),
base_(address.base.encoding()),
scale_(TimesOne),
index_(Registers::Invalid),
disp_(address.offset) {}
explicit Operand(const BaseIndex& address)
: kind_(MEM_SCALE),
base_(address.base.encoding()),
scale_(address.scale),
index_(address.index.encoding()),
disp_(address.offset) {}
Operand(Register base, Register index, Scale scale, int32_t disp = 0)
: kind_(MEM_SCALE),
base_(base.encoding()),
scale_(scale),
index_(index.encoding()),
disp_(disp) {}
Operand(Register reg, int32_t disp)
: kind_(MEM_REG_DISP),
base_(reg.encoding()),
scale_(TimesOne),
index_(Registers::Invalid),
disp_(disp) {}
explicit Operand(AbsoluteAddress address)
: kind_(MEM_ADDRESS32),
base_(Registers::Invalid),
scale_(TimesOne),
index_(Registers::Invalid),
disp_(X86Encoding::AddressImmediate(address.addr)) {}
explicit Operand(PatchedAbsoluteAddress address)
: kind_(MEM_ADDRESS32),
base_(Registers::Invalid),
scale_(TimesOne),
index_(Registers::Invalid),
disp_(X86Encoding::AddressImmediate(address.addr)) {}
Address toAddress() const {
MOZ_ASSERT(kind() == MEM_REG_DISP);
return Address(Register::FromCode(base()), disp());
}
BaseIndex toBaseIndex() const {
MOZ_ASSERT(kind() == MEM_SCALE);
return BaseIndex(Register::FromCode(base()), Register::FromCode(index()),
scale(), disp());
}
Kind kind() const { return kind_; }
Register::Encoding reg() const {
MOZ_ASSERT(kind() == REG);
return Register::Encoding(base_);
}
Register::Encoding base() const {
MOZ_ASSERT(kind() == MEM_REG_DISP || kind() == MEM_SCALE);
return Register::Encoding(base_);
}
Register::Encoding index() const {
MOZ_ASSERT(kind() == MEM_SCALE);
return index_;
}
Scale scale() const {
MOZ_ASSERT(kind() == MEM_SCALE);
return scale_;
}
FloatRegister::Encoding fpu() const {
MOZ_ASSERT(kind() == FPREG);
return FloatRegister::Encoding(base_);
}
int32_t disp() const {
MOZ_ASSERT(kind() == MEM_REG_DISP || kind() == MEM_SCALE);
return disp_;
}
void* address() const {
MOZ_ASSERT(kind() == MEM_ADDRESS32);
return reinterpret_cast<void*>(disp_);
}
bool containsReg(Register r) const {
switch (kind()) {
case REG:
return r.encoding() == reg();
case MEM_REG_DISP:
return r.encoding() == base();
case MEM_SCALE:
return r.encoding() == base() || r.encoding() == index();
default:
return false;
}
}
};
class CPUInfo {
public:
// As the SSE's were introduced in order, the presence of a later SSE implies
// the presence of an earlier SSE. For example, SSE4_2 support implies SSE2
// support.
enum SSEVersion {
UnknownSSE = 0,
NoSSE = 1,
SSE = 2,
SSE2 = 3,
SSE3 = 4,
SSSE3 = 5,
SSE4_1 = 6,
SSE4_2 = 7
};
static const int AVX_PRESENT_BIT = 8;
static SSEVersion GetSSEVersion() {
MOZ_ASSERT(FlagsHaveBeenComputed());
MOZ_ASSERT_IF(maxEnabledSSEVersion != UnknownSSE,
maxSSEVersion <= maxEnabledSSEVersion);
return maxSSEVersion;
}
static bool IsAVXPresent() {
MOZ_ASSERT(FlagsHaveBeenComputed());
MOZ_ASSERT_IF(!avxEnabled, !avxPresent);
return avxPresent;
}
static inline uint32_t GetFingerprint() {
return GetSSEVersion() | (IsAVXPresent() ? AVX_PRESENT_BIT : 0);
}
private:
static SSEVersion maxSSEVersion;
static SSEVersion maxEnabledSSEVersion;
static bool avxPresent;
static bool avxEnabled;
static bool popcntPresent;
static bool bmi1Present;
static bool bmi2Present;
static bool lzcntPresent;
static bool fmaPresent;
static bool avx2Present;
static bool f16cPresent;
static void SetMaxEnabledSSEVersion(SSEVersion v) {
if (maxEnabledSSEVersion == UnknownSSE) {
maxEnabledSSEVersion = v;
} else {
maxEnabledSSEVersion = std::min(v, maxEnabledSSEVersion);
}
}
public:
static bool IsSSE2Present() {
#ifdef JS_CODEGEN_X64
return true;
#else
return GetSSEVersion() >= SSE2;
#endif
}
static bool IsSSE3Present() { return GetSSEVersion() >= SSE3; }
static bool IsSSSE3Present() { return GetSSEVersion() >= SSSE3; }
static bool IsSSE41Present() { return GetSSEVersion() >= SSE4_1; }
static bool IsSSE42Present() { return GetSSEVersion() >= SSE4_2; }
static bool IsPOPCNTPresent() { return popcntPresent; }
static bool IsBMI1Present() { return bmi1Present; }
static bool IsBMI2Present() { return bmi2Present; }
static bool IsLZCNTPresent() { return lzcntPresent; }
static bool IsFMAPresent() { return fmaPresent; }
static bool IsAVX2Present() { return avx2Present; }
static bool IsF16CPresent() { return f16cPresent; }
static bool FlagsHaveBeenComputed() { return maxSSEVersion != UnknownSSE; }
static void ComputeFlags();
// The following should be called only before JS_Init (where the flags are
// computed). If several are called, the most restrictive setting is kept.
static void SetSSE3Disabled() {
MOZ_ASSERT(!FlagsHaveBeenComputed());
SetMaxEnabledSSEVersion(SSE2);
avxEnabled = false;
}
static void SetSSSE3Disabled() {
MOZ_ASSERT(!FlagsHaveBeenComputed());
SetMaxEnabledSSEVersion(SSE3);
avxEnabled = false;
}
static void SetSSE41Disabled() {
MOZ_ASSERT(!FlagsHaveBeenComputed());
SetMaxEnabledSSEVersion(SSSE3);
avxEnabled = false;
}
static void SetSSE42Disabled() {
MOZ_ASSERT(!FlagsHaveBeenComputed());
SetMaxEnabledSSEVersion(SSE4_1);
avxEnabled = false;
}
static void SetAVXDisabled() {
MOZ_ASSERT(!FlagsHaveBeenComputed());
avxEnabled = false;
}
static void SetAVXEnabled() {
MOZ_ASSERT(!FlagsHaveBeenComputed());
MOZ_ASSERT(maxEnabledSSEVersion == UnknownSSE,
"Can't enable AVX when SSE has been restricted");
avxEnabled = true;
}
};
class AssemblerX86Shared : public AssemblerShared {
protected:
struct RelativePatch {
int32_t offset;
void* target;
RelocationKind kind;
RelativePatch(int32_t offset, void* target, RelocationKind kind)
: offset(offset), target(target), kind(kind) {}
};
CompactBufferWriter jumpRelocations_;
CompactBufferWriter dataRelocations_;
void writeDataRelocation(ImmGCPtr ptr) {
// Raw GC pointer relocations and Value relocations both end up in
// Assembler::TraceDataRelocations.
if (ptr.value) {
if (gc::IsInsideNursery(ptr.value)) {
embedsNurseryPointers_ = true;
}
dataRelocations_.writeUnsigned(masm.currentOffset());
}
}
protected:
X86Encoding::BaseAssemblerSpecific masm;
using JmpSrc = X86Encoding::JmpSrc;
using JmpDst = X86Encoding::JmpDst;
public:
AssemblerX86Shared() {
if (!HasAVX()) {
masm.disableVEX();
}
}
enum Condition {
Equal = X86Encoding::ConditionE,
NotEqual = X86Encoding::ConditionNE,
Above = X86Encoding::ConditionA,
AboveOrEqual = X86Encoding::ConditionAE,
Below = X86Encoding::ConditionB,
BelowOrEqual = X86Encoding::ConditionBE,
GreaterThan = X86Encoding::ConditionG,
GreaterThanOrEqual = X86Encoding::ConditionGE,
LessThan = X86Encoding::ConditionL,
LessThanOrEqual = X86Encoding::ConditionLE,
Overflow = X86Encoding::ConditionO,
NoOverflow = X86Encoding::ConditionNO,
CarrySet = X86Encoding::ConditionC,
CarryClear = X86Encoding::ConditionNC,
Signed = X86Encoding::ConditionS,
NotSigned = X86Encoding::ConditionNS,
Zero = X86Encoding::ConditionE,
NonZero = X86Encoding::ConditionNE,
Parity = X86Encoding::ConditionP,
NoParity = X86Encoding::ConditionNP
};
enum class SSERoundingMode {
Nearest = int(X86Encoding::SSERoundingMode::RoundToNearest),
Floor = int(X86Encoding::SSERoundingMode::RoundDown),
Ceil = int(X86Encoding::SSERoundingMode::RoundUp),
Trunc = int(X86Encoding::SSERoundingMode::RoundToZero)
};
// If this bit is set, the vucomisd operands have to be inverted.
static const int DoubleConditionBitInvert = 0x10;
// Bit set when a DoubleCondition does not map to a single x86 condition.
// The macro assembler has to special-case these conditions.
static const int DoubleConditionBitSpecial = 0x20;
static const int DoubleConditionBits =
DoubleConditionBitInvert | DoubleConditionBitSpecial;
enum DoubleCondition {
// These conditions will only evaluate to true if the comparison is ordered
// - i.e. neither operand is NaN.
DoubleOrdered = NoParity,
DoubleEqual = Equal | DoubleConditionBitSpecial,
DoubleNotEqual = NotEqual,
DoubleGreaterThan = Above,
DoubleGreaterThanOrEqual = AboveOrEqual,
DoubleLessThan = Above | DoubleConditionBitInvert,
DoubleLessThanOrEqual = AboveOrEqual | DoubleConditionBitInvert,
// If either operand is NaN, these conditions always evaluate to true.
DoubleUnordered = Parity,
DoubleEqualOrUnordered = Equal,
DoubleNotEqualOrUnordered = NotEqual | DoubleConditionBitSpecial,
DoubleGreaterThanOrUnordered = Below | DoubleConditionBitInvert,
DoubleGreaterThanOrEqualOrUnordered =
BelowOrEqual | DoubleConditionBitInvert,
DoubleLessThanOrUnordered = Below,
DoubleLessThanOrEqualOrUnordered = BelowOrEqual
};
enum NaNCond { NaN_HandledByCond, NaN_IsTrue, NaN_IsFalse };
// If the primary condition returned by ConditionFromDoubleCondition doesn't
// handle NaNs properly, return NaN_IsFalse if the comparison should be
// overridden to return false on NaN, NaN_IsTrue if it should be overridden
// to return true on NaN, or NaN_HandledByCond if no secondary check is
// needed.
static inline NaNCond NaNCondFromDoubleCondition(DoubleCondition cond) {
switch (cond) {
case DoubleOrdered:
case DoubleNotEqual:
case DoubleGreaterThan:
case DoubleGreaterThanOrEqual:
case DoubleLessThan:
case DoubleLessThanOrEqual:
case DoubleUnordered:
case DoubleEqualOrUnordered:
case DoubleGreaterThanOrUnordered:
case DoubleGreaterThanOrEqualOrUnordered:
case DoubleLessThanOrUnordered:
case DoubleLessThanOrEqualOrUnordered:
return NaN_HandledByCond;
case DoubleEqual:
return NaN_IsFalse;
case DoubleNotEqualOrUnordered:
return NaN_IsTrue;
}
MOZ_CRASH("Unknown double condition");
}
static void StaticAsserts() {
// DoubleConditionBits should not interfere with x86 condition codes.
static_assert(!((Equal | NotEqual | Above | AboveOrEqual | Below |
BelowOrEqual | Parity | NoParity) &
DoubleConditionBits));
}
static Condition InvertCondition(Condition cond);
static Condition UnsignedCondition(Condition cond);
static Condition ConditionWithoutEqual(Condition cond);
static DoubleCondition InvertCondition(DoubleCondition cond);
// Return the primary condition to test. Some primary conditions may not
// handle NaNs properly and may therefore require a secondary condition.
// Use NaNCondFromDoubleCondition to determine what else is needed.
static inline Condition ConditionFromDoubleCondition(DoubleCondition cond) {
return static_cast<Condition>(cond & ~DoubleConditionBits);
}
static void TraceDataRelocations(JSTracer* trc, JitCode* code,
CompactBufferReader& reader);
void setUnlimitedBuffer() {
// No-op on this platform
}
bool oom() const {
return AssemblerShared::oom() || masm.oom() || jumpRelocations_.oom() ||
dataRelocations_.oom();
}
bool reserve(size_t size) { return masm.reserve(size); }
bool swapBuffer(wasm::Bytes& other) { return masm.swapBuffer(other); }
void setPrinter(Sprinter* sp) { masm.setPrinter(sp); }
Register getStackPointer() const { return StackPointer; }
void executableCopy(void* buffer);
void processCodeLabels(uint8_t* rawCode);
void copyJumpRelocationTable(uint8_t* dest);
void copyDataRelocationTable(uint8_t* dest);
// Size of the instruction stream, in bytes.
size_t size() const { return masm.size(); }
// Size of the jump relocation table, in bytes.
size_t jumpRelocationTableBytes() const { return jumpRelocations_.length(); }
size_t dataRelocationTableBytes() const { return dataRelocations_.length(); }
// Size of the data table, in bytes.
size_t bytesNeeded() const {
return size() + jumpRelocationTableBytes() + dataRelocationTableBytes();
}
public:
void haltingAlign(int alignment) {
MOZ_ASSERT(hasCreator());
masm.haltingAlign(alignment);
}
void nopAlign(int alignment) {
MOZ_ASSERT(hasCreator());
masm.nopAlign(alignment);
}
void writeCodePointer(CodeLabel* label) {
MOZ_ASSERT(hasCreator());
// Use -1 as dummy value. This will be patched after codegen.
masm.jumpTablePointer(-1);
label->patchAt()->bind(masm.size());
}
void cmovCCl(Condition cond, const Operand& src, Register dest) {
X86Encoding::Condition cc = static_cast<X86Encoding::Condition>(cond);
switch (src.kind()) {
case Operand::REG:
masm.cmovCCl_rr(cc, src.reg(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.cmovCCl_mr(cc, src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.cmovCCl_mr(cc, src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void cmovCCl(Condition cond, Register src, Register dest) {
X86Encoding::Condition cc = static_cast<X86Encoding::Condition>(cond);
masm.cmovCCl_rr(cc, src.encoding(), dest.encoding());
}
void cmovzl(const Operand& src, Register dest) {
cmovCCl(Condition::Zero, src, dest);
}
void cmovnzl(const Operand& src, Register dest) {
cmovCCl(Condition::NonZero, src, dest);
}
void movl(Imm32 imm32, Register dest) {
MOZ_ASSERT(hasCreator());
masm.movl_i32r(imm32.value, dest.encoding());
}
void movl(Register src, Register dest) {
MOZ_ASSERT(hasCreator());
masm.movl_rr(src.encoding(), dest.encoding());
}
void movl(const Operand& src, Register dest) {
MOZ_ASSERT(hasCreator());
switch (src.kind()) {
case Operand::REG:
masm.movl_rr(src.reg(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.movl_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.movl_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
case Operand::MEM_ADDRESS32:
masm.movl_mr(src.address(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void movl(Register src, const Operand& dest) {
MOZ_ASSERT(hasCreator());
switch (dest.kind()) {
case Operand::REG:
masm.movl_rr(src.encoding(), dest.reg());
break;
case Operand::MEM_REG_DISP:
masm.movl_rm(src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.movl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
case Operand::MEM_ADDRESS32:
masm.movl_rm(src.encoding(), dest.address());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void movl(Imm32 imm32, const Operand& dest) {
switch (dest.kind()) {
case Operand::REG:
masm.movl_i32r(imm32.value, dest.reg());
break;
case Operand::MEM_REG_DISP:
masm.movl_i32m(imm32.value, dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.movl_i32m(imm32.value, dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
case Operand::MEM_ADDRESS32:
masm.movl_i32m(imm32.value, dest.address());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void xchgl(Register src, Register dest) {
masm.xchgl_rr(src.encoding(), dest.encoding());
}
void vmovapd(FloatRegister src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vmovapd_rr(src.encoding(), dest.encoding());
}
// Eventually vmovapd should be overloaded to support loads and
// stores too.
void vmovapd(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src.kind()) {
case Operand::FPREG:
masm.vmovapd_rr(src.fpu(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovaps(FloatRegister src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vmovaps_rr(src.encoding(), dest.encoding());
}
void vmovaps(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src.kind()) {
case Operand::MEM_REG_DISP:
masm.vmovaps_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vmovaps_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
case Operand::FPREG:
masm.vmovaps_rr(src.fpu(), dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovaps(FloatRegister src, const Operand& dest) {
MOZ_ASSERT(HasSSE2());
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
masm.vmovaps_rm(src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.vmovaps_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovups(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src.kind()) {
case Operand::MEM_REG_DISP:
masm.vmovups_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vmovups_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovups(FloatRegister src, const Operand& dest) {
MOZ_ASSERT(HasSSE2());
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
masm.vmovups_rm(src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.vmovups_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovsd(const Address& src, FloatRegister dest) {
masm.vmovsd_mr(src.offset, src.base.encoding(), dest.encoding());
}
void vmovsd(const BaseIndex& src, FloatRegister dest) {
masm.vmovsd_mr(src.offset, src.base.encoding(), src.index.encoding(),
src.scale, dest.encoding());
}
void vmovsd(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(hasCreator());
switch (src.kind()) {
case Operand::MEM_REG_DISP:
vmovsd(src.toAddress(), dest);
break;
case Operand::MEM_SCALE:
vmovsd(src.toBaseIndex(), dest);
break;
default:
MOZ_CRASH("Unknown operand for vmovsd");
}
}
void vmovsd(FloatRegister src, const Address& dest) {
masm.vmovsd_rm(src.encoding(), dest.offset, dest.base.encoding());
}
void vmovsd(FloatRegister src, const BaseIndex& dest) {
masm.vmovsd_rm(src.encoding(), dest.offset, dest.base.encoding(),
dest.index.encoding(), dest.scale);
}
// Note special semantics of this - does not clobber high bits of destination.
void vmovsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
masm.vmovsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vmovss(const Address& src, FloatRegister dest) {
masm.vmovss_mr(src.offset, src.base.encoding(), dest.encoding());
}
void vmovss(const BaseIndex& src, FloatRegister dest) {
masm.vmovss_mr(src.offset, src.base.encoding(), src.index.encoding(),
src.scale, dest.encoding());
}
void vmovss(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(hasCreator());
switch (src.kind()) {
case Operand::MEM_REG_DISP:
vmovss(src.toAddress(), dest);
break;
case Operand::MEM_SCALE:
vmovss(src.toBaseIndex(), dest);
break;
default:
MOZ_CRASH("Unknown operand for vmovss");
}
}
void vmovss(FloatRegister src, const Address& dest) {
masm.vmovss_rm(src.encoding(), dest.offset, dest.base.encoding());
}
void vmovss(FloatRegister src, const BaseIndex& dest) {
masm.vmovss_rm(src.encoding(), dest.offset, dest.base.encoding(),
dest.index.encoding(), dest.scale);
}
void vmovss(FloatRegister src, const Operand& dest) {
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
vmovss(src, dest.toAddress());
break;
case Operand::MEM_SCALE:
vmovss(src, dest.toBaseIndex());
break;
default:
MOZ_CRASH("Unknown operand for vmovss");
}
}
// Note special semantics of this - does not clobber high bits of destination.
void vmovss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
masm.vmovss_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vmovdqu(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
MOZ_ASSERT(hasCreator());
switch (src.kind()) {
case Operand::MEM_REG_DISP:
masm.vmovdqu_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vmovdqu_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovdqu(FloatRegister src, const Operand& dest) {
MOZ_ASSERT(HasSSE2());
MOZ_ASSERT(hasCreator());
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
masm.vmovdqu_rm(src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.vmovdqu_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovdqa(const Operand& src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
switch (src.kind()) {
case Operand::FPREG:
masm.vmovdqa_rr(src.fpu(), dest.encoding());
break;
case Operand::MEM_REG_DISP:
masm.vmovdqa_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.vmovdqa_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovdqa(FloatRegister src, const Operand& dest) {
MOZ_ASSERT(HasSSE2());
switch (dest.kind()) {
case Operand::MEM_REG_DISP:
masm.vmovdqa_rm(src.encoding(), dest.disp(), dest.base());
break;
case Operand::MEM_SCALE:
masm.vmovdqa_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
dest.scale());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void vmovdqa(FloatRegister src, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vmovdqa_rr(src.encoding(), dest.encoding());
}
void vcvtss2sd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vcvtss2sd_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void vcvtsd2ss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
MOZ_ASSERT(HasSSE2());
masm.vcvtsd2ss_rr(src1.encoding(), src0.encoding(), dest.encoding());
}
void movzbl(const Operand& src, Register dest) {
switch (src.kind()) {
case Operand::MEM_REG_DISP:
masm.movzbl_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.movzbl_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default:
MOZ_CRASH("unexpected operand kind");
}
}
void movsbl(Register src, Register dest) {
masm.movsbl_rr(src.encoding(), dest.encoding());
}
void movsbl(const Operand& src, Register dest) {
switch (src.kind()) {
case Operand::MEM_REG_DISP:
masm.movsbl_mr(src.disp(), src.base(), dest.encoding());
break;
case Operand::MEM_SCALE:
masm.movsbl_mr(src.disp(), src.base(), src.index(), src.scale(),
dest.encoding());
break;
default: