Assembler-arm.h - mozsearch

mozilla-central/js/src/jit/arm/Assembler-arm.h

Enable keyboard shortcuts

Source code

Revision control

Copy as Markdown

Other Tools

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-

 * vim: set ts=8 sts=2 et sw=2 tw=80:

 * This Source Code Form is subject to the terms of the Mozilla Public

 * License, v. 2.0. If a copy of the MPL was not distributed with this

 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifndef jit_arm_Assembler_arm_h

#define jit_arm_Assembler_arm_h

#include "mozilla/Attributes.h"

#include "mozilla/MathAlgorithms.h"

#include <algorithm>

#include <iterator>

#include <type_traits>

#include "jit/arm/Architecture-arm.h"

#include "jit/arm/disasm/Disasm-arm.h"

#include "jit/CompactBuffer.h"

#include "jit/JitCode.h"

#include "jit/shared/Assembler-shared.h"

#include "jit/shared/Disassembler-shared.h"

#include "jit/shared/IonAssemblerBufferWithConstantPools.h"

#include "wasm/WasmTypeDecls.h"

union PoolHintPun;

namespace js {

namespace jit {

using LiteralDoc = DisassemblerSpew::LiteralDoc;

using LabelDoc = DisassemblerSpew::LabelDoc;

// NOTE: there are duplicates in this list! Sometimes we want to specifically

// refer to the link register as a link register (bl lr is much clearer than bl

// r14). HOWEVER, this register can easily be a gpr when it is not busy holding

// the return address.

static constexpr Register r0{Registers::r0};

static constexpr Register r1{Registers::r1};

static constexpr Register r2{Registers::r2};

static constexpr Register r3{Registers::r3};

static constexpr Register r4{Registers::r4};

static constexpr Register r5{Registers::r5};

static constexpr Register r6{Registers::r6};

static constexpr Register r7{Registers::r7};

static constexpr Register r8{Registers::r8};

static constexpr Register r9{Registers::r9};

static constexpr Register r10{Registers::r10};

static constexpr Register r11{Registers::r11};

static constexpr Register r12{Registers::ip};

static constexpr Register ip{Registers::ip};

static constexpr Register sp{Registers::sp};

static constexpr Register r14{Registers::lr};

static constexpr Register lr{Registers::lr};

static constexpr Register pc{Registers::pc};

static constexpr Register ScratchRegister{Registers::ip};

// Helper class for ScratchRegister usage. Asserts that only one piece

// of code thinks it has exclusive ownership of the scratch register.

struct ScratchRegisterScope : public AutoRegisterScope {

  explicit ScratchRegisterScope(MacroAssembler& masm)

      : AutoRegisterScope(masm, ScratchRegister) {}

};

struct SecondScratchRegisterScope : public AutoRegisterScope {

  explicit SecondScratchRegisterScope(MacroAssembler& masm);

};

static constexpr Register OsrFrameReg = r3;

static constexpr Register CallTempReg0 = r5;

static constexpr Register CallTempReg1 = r6;

static constexpr Register CallTempReg2 = r7;

static constexpr Register CallTempReg3 = r8;

static constexpr Register CallTempReg4 = r0;

static constexpr Register CallTempReg5 = r1;

static constexpr Register IntArgReg0 = r0;

static constexpr Register IntArgReg1 = r1;

static constexpr Register IntArgReg2 = r2;

static constexpr Register IntArgReg3 = r3;

static constexpr Register HeapReg = r10;

static constexpr Register CallTempNonArgRegs[] = {r5, r6, r7, r8};

static const uint32_t NumCallTempNonArgRegs = std::size(CallTempNonArgRegs);

// These register assignments for the 64-bit atomic ops are frequently too

// constraining, but we have no way of expressing looser constraints to the

// register allocator.

// CompareExchange: Any two odd/even pairs would do for `new` and `out`, and any

// pair would do for `old`, so long as none of them overlap.

static constexpr Register CmpXchgOldLo = r4;

static constexpr Register CmpXchgOldHi = r5;

static constexpr Register64 CmpXchgOld64 =

    Register64(CmpXchgOldHi, CmpXchgOldLo);

static constexpr Register CmpXchgNewLo = IntArgReg2;

static constexpr Register CmpXchgNewHi = IntArgReg3;

static constexpr Register64 CmpXchgNew64 =

    Register64(CmpXchgNewHi, CmpXchgNewLo);

static constexpr Register CmpXchgOutLo = IntArgReg0;

static constexpr Register CmpXchgOutHi = IntArgReg1;

static constexpr Register64 CmpXchgOut64 =

    Register64(CmpXchgOutHi, CmpXchgOutLo);

// Exchange: Any two non-equal odd/even pairs would do for `new` and `out`.

static constexpr Register XchgNewLo = IntArgReg2;

static constexpr Register XchgNewHi = IntArgReg3;

static constexpr Register64 XchgNew64 = Register64(XchgNewHi, XchgNewLo);

static constexpr Register XchgOutLo = IntArgReg0;

static constexpr Register XchgOutHi = IntArgReg1;

// Atomic rmw operations: Any two odd/even pairs would do for `tmp` and `out`,

// and any pair would do for `val`, so long as none of them overlap.

static constexpr Register FetchOpValLo = r4;

static constexpr Register FetchOpValHi = r5;

static constexpr Register64 FetchOpVal64 =

    Register64(FetchOpValHi, FetchOpValLo);

static constexpr Register FetchOpTmpLo = IntArgReg2;

static constexpr Register FetchOpTmpHi = IntArgReg3;

static constexpr Register64 FetchOpTmp64 =

    Register64(FetchOpTmpHi, FetchOpTmpLo);

static constexpr Register FetchOpOutLo = IntArgReg0;

static constexpr Register FetchOpOutHi = IntArgReg1;

static constexpr Register64 FetchOpOut64 =

    Register64(FetchOpOutHi, FetchOpOutLo);

class ABIArgGenerator {

  unsigned intRegIndex_;

  unsigned floatRegIndex_;

  uint32_t stackOffset_;

  ABIArg current_;

  // ARM can either use HardFp (use float registers for float arguments), or

  // SoftFp (use general registers for float arguments) ABI.  We keep this

  // switch as a runtime switch because wasm always use the HardFp back-end

  // while the calls to native functions have to use the one provided by the

  // system.

  bool useHardFp_;

  ABIArg softNext(MIRType argType);

  ABIArg hardNext(MIRType argType);

 public:

  ABIArgGenerator();

  void setUseHardFp(bool useHardFp) {

    MOZ_ASSERT(intRegIndex_ == 0 && floatRegIndex_ == 0);

    useHardFp_ = useHardFp;

  ABIArg next(MIRType argType);

  ABIArg& current() { return current_; }

  uint32_t stackBytesConsumedSoFar() const { return stackOffset_; }

  void increaseStackOffset(uint32_t bytes) { stackOffset_ += bytes; }

};

bool IsUnaligned(const wasm::MemoryAccessDesc& access);

// These registers may be volatile or nonvolatile.

static constexpr Register ABINonArgReg0 = r4;

static constexpr Register ABINonArgReg1 = r5;

static constexpr Register ABINonArgReg2 = r6;

static constexpr Register ABINonArgReg3 = r7;

// This register may be volatile or nonvolatile. Avoid d15 which is the

// ScratchDoubleReg_.

static constexpr FloatRegister ABINonArgDoubleReg{FloatRegisters::d8,

                                                  VFPRegister::Double};

// These registers may be volatile or nonvolatile.

// Note: these three registers are all guaranteed to be different

static constexpr Register ABINonArgReturnReg0 = r4;

static constexpr Register ABINonArgReturnReg1 = r5;

static constexpr Register ABINonVolatileReg = r6;

// This register is guaranteed to be clobberable during the prologue and

// epilogue of an ABI call which must preserve both ABI argument, return

// and non-volatile registers.

static constexpr Register ABINonArgReturnVolatileReg = lr;

// Instance pointer argument register for WebAssembly functions. This must not

// alias any other register used for passing function arguments or return

// values. Preserved by WebAssembly functions.

static constexpr Register InstanceReg = r9;

// Registers used for wasm table calls. These registers must be disjoint

// from the ABI argument registers, InstanceReg and each other.

static constexpr Register WasmTableCallScratchReg0 = ABINonArgReg0;

static constexpr Register WasmTableCallScratchReg1 = ABINonArgReg1;

static constexpr Register WasmTableCallSigReg = ABINonArgReg2;

static constexpr Register WasmTableCallIndexReg = ABINonArgReg3;

// Registers used for ref calls.

static constexpr Register WasmCallRefCallScratchReg0 = ABINonArgReg0;

static constexpr Register WasmCallRefCallScratchReg1 = ABINonArgReg1;

static constexpr Register WasmCallRefReg = ABINonArgReg3;

// Register used as a scratch along the return path in the fast js -> wasm stub

// code.  This must not overlap ReturnReg, JSReturnOperand, or InstanceReg.

// It must be a volatile register.

static constexpr Register WasmJitEntryReturnScratch = r5;

static constexpr Register PreBarrierReg = r1;

static constexpr Register InterpreterPCReg = r9;

static constexpr Register InvalidReg{Registers::invalid_reg};

static constexpr FloatRegister InvalidFloatReg;

static constexpr Register JSReturnReg_Type = r3;

static constexpr Register JSReturnReg_Data = r2;

static constexpr Register StackPointer = sp;

static constexpr Register FramePointer = r11;

static constexpr Register ReturnReg = r0;

static constexpr Register64 ReturnReg64(r1, r0);

// The attribute '__value_in_regs' alters the calling convention of a function

// so that a structure of up to four elements can be returned via the argument

// registers rather than being written to memory.

static constexpr Register ReturnRegVal0 = IntArgReg0;

static constexpr Register ReturnRegVal1 = IntArgReg1;

static constexpr Register ReturnRegVal2 = IntArgReg2;

static constexpr Register ReturnRegVal3 = IntArgReg3;

static constexpr FloatRegister ReturnFloat32Reg = {FloatRegisters::d0,

                                                   VFPRegister::Single};

static constexpr FloatRegister ReturnDoubleReg = {FloatRegisters::d0,

                                                  VFPRegister::Double};

static constexpr FloatRegister ReturnSimd128Reg = InvalidFloatReg;

static constexpr FloatRegister ScratchFloat32Reg_ = {FloatRegisters::s30,

                                                     VFPRegister::Single};

static constexpr FloatRegister ScratchDoubleReg_ = {FloatRegisters::d15,

                                                    VFPRegister::Double};

static constexpr FloatRegister ScratchSimd128Reg = InvalidFloatReg;

static constexpr FloatRegister ScratchUIntReg = {FloatRegisters::d15,

                                                 VFPRegister::UInt};

static constexpr FloatRegister ScratchIntReg = {FloatRegisters::d15,

                                                VFPRegister::Int};

// Do not reference ScratchFloat32Reg_ directly, use ScratchFloat32Scope

// instead.

struct ScratchFloat32Scope : public AutoFloatRegisterScope {

  explicit ScratchFloat32Scope(MacroAssembler& masm)

      : AutoFloatRegisterScope(masm, ScratchFloat32Reg_) {}

};

// Do not reference ScratchDoubleReg_ directly, use ScratchDoubleScope instead.

struct ScratchDoubleScope : public AutoFloatRegisterScope {

  explicit ScratchDoubleScope(MacroAssembler& masm)

      : AutoFloatRegisterScope(masm, ScratchDoubleReg_) {}

};

// Registers used by RegExpMatcher and RegExpExecMatch stubs (do not use

// JSReturnOperand).

static constexpr Register RegExpMatcherRegExpReg = CallTempReg0;

static constexpr Register RegExpMatcherStringReg = CallTempReg1;

static constexpr Register RegExpMatcherLastIndexReg = CallTempReg2;

// Registers used by RegExpExecTest stub (do not use ReturnReg).

static constexpr Register RegExpExecTestRegExpReg = CallTempReg0;

static constexpr Register RegExpExecTestStringReg = CallTempReg1;

// Registers used by RegExpSearcher stub (do not use ReturnReg).

static constexpr Register RegExpSearcherRegExpReg = CallTempReg0;

static constexpr Register RegExpSearcherStringReg = CallTempReg1;

static constexpr Register RegExpSearcherLastIndexReg = CallTempReg2;

static constexpr FloatRegister d0 = {FloatRegisters::d0, VFPRegister::Double};

static constexpr FloatRegister d1 = {FloatRegisters::d1, VFPRegister::Double};

static constexpr FloatRegister d2 = {FloatRegisters::d2, VFPRegister::Double};

static constexpr FloatRegister d3 = {FloatRegisters::d3, VFPRegister::Double};

static constexpr FloatRegister d4 = {FloatRegisters::d4, VFPRegister::Double};

static constexpr FloatRegister d5 = {FloatRegisters::d5, VFPRegister::Double};

static constexpr FloatRegister d6 = {FloatRegisters::d6, VFPRegister::Double};

static constexpr FloatRegister d7 = {FloatRegisters::d7, VFPRegister::Double};

static constexpr FloatRegister d8 = {FloatRegisters::d8, VFPRegister::Double};

static constexpr FloatRegister d9 = {FloatRegisters::d9, VFPRegister::Double};

static constexpr FloatRegister d10 = {FloatRegisters::d10, VFPRegister::Double};

static constexpr FloatRegister d11 = {FloatRegisters::d11, VFPRegister::Double};

static constexpr FloatRegister d12 = {FloatRegisters::d12, VFPRegister::Double};

static constexpr FloatRegister d13 = {FloatRegisters::d13, VFPRegister::Double};

static constexpr FloatRegister d14 = {FloatRegisters::d14, VFPRegister::Double};

static constexpr FloatRegister d15 = {FloatRegisters::d15, VFPRegister::Double};

// For maximal awesomeness, 8 should be sufficent. ldrd/strd (dual-register

// load/store) operate in a single cycle when the address they are dealing with

// is 8 byte aligned. Also, the ARM abi wants the stack to be 8 byte aligned at

// function boundaries. I'm trying to make sure this is always true.

static constexpr uint32_t ABIStackAlignment = 8;

static constexpr uint32_t CodeAlignment = 8;

static constexpr uint32_t JitStackAlignment = 8;

static constexpr uint32_t JitStackValueAlignment =

    JitStackAlignment / sizeof(Value);

static_assert(JitStackAlignment % sizeof(Value) == 0 &&

                  JitStackValueAlignment >= 1,

              "Stack alignment should be a non-zero multiple of sizeof(Value)");

static constexpr uint32_t SimdMemoryAlignment = 8;

static_assert(CodeAlignment % SimdMemoryAlignment == 0,

              "Code alignment should be larger than any of the alignments "

              "which are used for "

              "the constant sections of the code buffer.  Thus it should be "

              "larger than the "

              "alignment for SIMD constants.");

static_assert(JitStackAlignment % SimdMemoryAlignment == 0,

              "Stack alignment should be larger than any of the alignments "

              "which are used for "

              "spilled values.  Thus it should be larger than the alignment "

              "for SIMD accesses.");

static const uint32_t WasmStackAlignment = SimdMemoryAlignment;

static const uint32_t WasmTrapInstructionLength = 4;

// See comments in wasm::GenerateFunctionPrologue.  The difference between these

// is the size of the largest callable prologue on the platform.

static constexpr uint32_t WasmCheckedCallEntryOffset = 0u;

static const Scale ScalePointer = TimesFour;

class Instruction;

class InstBranchImm;

uint32_t RM(Register r);

uint32_t RS(Register r);

uint32_t RD(Register r);

uint32_t RT(Register r);

uint32_t RN(Register r);

uint32_t maybeRD(Register r);

uint32_t maybeRT(Register r);

uint32_t maybeRN(Register r);

Register toRN(Instruction i);

Register toRM(Instruction i);

Register toRD(Instruction i);

Register toR(Instruction i);

class VFPRegister;

uint32_t VD(VFPRegister vr);

uint32_t VN(VFPRegister vr);

uint32_t VM(VFPRegister vr);

// For being passed into the generic vfp instruction generator when there is an

// instruction that only takes two registers.

static constexpr VFPRegister NoVFPRegister(VFPRegister::Double, 0, false, true);

struct ImmTag : public Imm32 {

  explicit ImmTag(JSValueTag mask) : Imm32(int32_t(mask)) {}

};

struct ImmType : public ImmTag {

  explicit ImmType(JSValueType type) : ImmTag(JSVAL_TYPE_TO_TAG(type)) {}

};

enum Index {

  Offset = 0 << 21 | 1 << 24,

  PreIndex = 1 << 21 | 1 << 24,

  PostIndex = 0 << 21 | 0 << 24

  // The docs were rather unclear on this. It sounds like

  // 1 << 21 | 0 << 24 encodes dtrt.

};

enum IsImmOp2_ { IsImmOp2 = 1 << 25, IsNotImmOp2 = 0 << 25 };

enum IsImmDTR_ { IsImmDTR = 0 << 25, IsNotImmDTR = 1 << 25 };

// For the extra memory operations, ldrd, ldrsb, ldrh.

enum IsImmEDTR_ { IsImmEDTR = 1 << 22, IsNotImmEDTR = 0 << 22 };

enum ShiftType {

  LSL = 0,   // << 5

  LSR = 1,   // << 5

  ASR = 2,   // << 5

  ROR = 3,   // << 5

  RRX = ROR  // RRX is encoded as ROR with a 0 offset.

};

// Modes for STM/LDM. Names are the suffixes applied to the instruction.

enum DTMMode {

  A = 0 << 24,  // empty / after

  B = 1 << 24,  // full / before

  D = 0 << 23,  // decrement

  I = 1 << 23,  // increment

  DA = D | A,

  DB = D | B,

  IA = I | A,

  IB = I | B

};

enum DTMWriteBack { WriteBack = 1 << 21, NoWriteBack = 0 << 21 };

// Condition code updating mode.

enum SBit {

  SetCC = 1 << 20,   // Set condition code.

  LeaveCC = 0 << 20  // Leave condition code unchanged.

};

enum LoadStore { IsLoad = 1 << 20, IsStore = 0 << 20 };

// You almost never want to use this directly. Instead, you wantto pass in a

// signed constant, and let this bit be implicitly set for you. This is however,

// necessary if we want a negative index.

enum IsUp_ { IsUp = 1 << 23, IsDown = 0 << 23 };

enum ALUOp {

  OpMov = 0xd << 21,

  OpMvn = 0xf << 21,

  OpAnd = 0x0 << 21,

  OpBic = 0xe << 21,

  OpEor = 0x1 << 21,

  OpOrr = 0xc << 21,

  OpAdc = 0x5 << 21,

  OpAdd = 0x4 << 21,

  OpSbc = 0x6 << 21,

  OpSub = 0x2 << 21,

  OpRsb = 0x3 << 21,

  OpRsc = 0x7 << 21,

  OpCmn = 0xb << 21,

  OpCmp = 0xa << 21,

  OpTeq = 0x9 << 21,

  OpTst = 0x8 << 21,

  OpInvalid = -1

};

enum MULOp {

  OpmMul = 0 << 21,

  OpmMla = 1 << 21,

  OpmUmaal = 2 << 21,

  OpmMls = 3 << 21,

  OpmUmull = 4 << 21,

  OpmUmlal = 5 << 21,

  OpmSmull = 6 << 21,

  OpmSmlal = 7 << 21

};

enum BranchTag {

  OpB = 0x0a000000,

  OpBMask = 0x0f000000,

  OpBDestMask = 0x00ffffff,

  OpBl = 0x0b000000,

  OpBlx = 0x012fff30,

  OpBx = 0x012fff10

};

// Just like ALUOp, but for the vfp instruction set.

enum VFPOp {

  OpvMul = 0x2 << 20,

  OpvAdd = 0x3 << 20,

  OpvSub = 0x3 << 20 | 0x1 << 6,

  OpvDiv = 0x8 << 20,

  OpvMov = 0xB << 20 | 0x1 << 6,

  OpvAbs = 0xB << 20 | 0x3 << 6,

  OpvNeg = 0xB << 20 | 0x1 << 6 | 0x1 << 16,

  OpvSqrt = 0xB << 20 | 0x3 << 6 | 0x1 << 16,

  OpvCmp = 0xB << 20 | 0x1 << 6 | 0x4 << 16,

  OpvCmpz = 0xB << 20 | 0x1 << 6 | 0x5 << 16

};

// Negate the operation, AND negate the immediate that we were passed in.

ALUOp ALUNeg(ALUOp op, Register dest, Register scratch, Imm32* imm,

             Register* negDest);

bool can_dbl(ALUOp op);

bool condsAreSafe(ALUOp op);

// If there is a variant of op that has a dest (think cmp/sub) return that

// variant of it.

ALUOp getDestVariant(ALUOp op);

static constexpr ValueOperand JSReturnOperand{JSReturnReg_Type,

                                              JSReturnReg_Data};

static const ValueOperand softfpReturnOperand = ValueOperand(r1, r0);

// All of these classes exist solely to shuffle data into the various operands.

// For example Operand2 can be an imm8, a register-shifted-by-a-constant or a

// register-shifted-by-a-register. We represent this in C++ by having a base

// class Operand2, which just stores the 32 bits of data as they will be encoded

// in the instruction. You cannot directly create an Operand2 since it is

// tricky, and not entirely sane to do so. Instead, you create one of its child

// classes, e.g. Imm8. Imm8's constructor takes a single integer argument. Imm8

// will verify that its argument can be encoded as an ARM 12 bit imm8, encode it

// using an Imm8data, and finally call its parent's (Operand2) constructor with

// the Imm8data. The Operand2 constructor will then call the Imm8data's encode()

// function to extract the raw bits from it.

//

// In the future, we should be able to extract data from the Operand2 by asking

// it for its component Imm8data structures. The reason this is so horribly

// round-about is we wanted to have Imm8 and RegisterShiftedRegister inherit

// directly from Operand2 but have all of them take up only a single word of

// storage. We also wanted to avoid passing around raw integers at all since

// they are error prone.

class Op2Reg;

class O2RegImmShift;

class O2RegRegShift;

namespace datastore {

class Reg {

  // The "second register".

  uint32_t rm_ : 4;

  // Do we get another register for shifting.

  uint32_t rrs_ : 1;

  uint32_t type_ : 2;

  // We'd like this to be a more sensible encoding, but that would need to be

  // a struct and that would not pack :(

  uint32_t shiftAmount_ : 5;

 protected:

  // Mark as a protected field to avoid unused private field warnings.

  uint32_t pad_ : 20;

 public:

  Reg(uint32_t rm, ShiftType type, uint32_t rsr, uint32_t shiftAmount)

      : rm_(rm), rrs_(rsr), type_(type), shiftAmount_(shiftAmount), pad_(0) {}

  explicit Reg(const Op2Reg& op) { memcpy(this, &op, sizeof(*this)); }

  uint32_t shiftAmount() const { return shiftAmount_; }

  uint32_t encode() const {

    return rm_ | (rrs_ << 4) | (type_ << 5) | (shiftAmount_ << 7);

};

// Op2 has a mode labelled "<imm8m>", which is arm's magical immediate encoding.

// Some instructions actually get 8 bits of data, which is called Imm8Data

// below. These should have edit distance > 1, but this is how it is for now.

class Imm8mData {

  uint32_t data_ : 8;

  uint32_t rot_ : 4;

 protected:

  // Mark as a protected field to avoid unused private field warnings.

  uint32_t buff_ : 19;

 private:

  // Throw in an extra bit that will be 1 if we can't encode this properly.

  // if we can encode it properly, a simple "|" will still suffice to meld it

  // into the instruction.

  uint32_t invalid_ : 1;

 public:

  // Default constructor makes an invalid immediate.

  Imm8mData() : data_(0xff), rot_(0xf), buff_(0), invalid_(true) {}

  Imm8mData(uint32_t data, uint32_t rot)

      : data_(data), rot_(rot), buff_(0), invalid_(false) {

    MOZ_ASSERT(data == data_);

    MOZ_ASSERT(rot == rot_);

  bool invalid() const { return invalid_; }

  uint32_t encode() const {

    MOZ_ASSERT(!invalid_);

    return data_ | (rot_ << 8);

};

};

class Imm8Data {

  uint32_t imm4L_ : 4;

 protected:

  // Mark as a protected field to avoid unused private field warnings.

  uint32_t pad_ : 4;

 private:

  uint32_t imm4H_ : 4;

 public:

  explicit Imm8Data(uint32_t imm) : imm4L_(imm & 0xf), imm4H_(imm >> 4) {

    MOZ_ASSERT(imm <= 0xff);

  uint32_t encode() const { return imm4L_ | (imm4H_ << 8); };

};

// VLDR/VSTR take an 8 bit offset, which is implicitly left shifted by 2.

class Imm8VFPOffData {

  uint32_t data_;

 public:

  explicit Imm8VFPOffData(uint32_t imm) : data_(imm) {

    MOZ_ASSERT((imm & ~(0xff)) == 0);

  uint32_t encode() const { return data_; };

};

// ARM can magically encode 256 very special immediates to be moved into a

// register.

struct Imm8VFPImmData {

  // This structure's members are public and it has no constructor to

  // initialize them, for a very special reason. Were this structure to

  // have a constructor, the initialization for DoubleEncoder's internal

  // table (see below) would require a rather large static constructor on

  // some of our supported compilers. The known solution to this is to mark

  // the constructor constexpr, but, again, some of our supported

  // compilers don't support constexpr! So we are reduced to public

  // members and eschewing a constructor in hopes that the initialization

  // of DoubleEncoder's table is correct.

  uint32_t imm4L : 4;

  uint32_t imm4H : 4;

  int32_t isInvalid : 24;

  uint32_t encode() const {

    // This assert is an attempting at ensuring that we don't create random

    // instances of this structure and then asking to encode() it.

    MOZ_ASSERT(isInvalid == 0);

    return imm4L | (imm4H << 16);

};

};

class Imm12Data {

  uint32_t data_ : 12;

 public:

  explicit Imm12Data(uint32_t imm) : data_(imm) { MOZ_ASSERT(data_ == imm); }

  uint32_t encode() const { return data_; }

};

class RIS {

  uint32_t shiftAmount_ : 5;

 public:

  explicit RIS(uint32_t imm) : shiftAmount_(imm) {

    MOZ_ASSERT(shiftAmount_ == imm);

  explicit RIS(Reg r) : shiftAmount_(r.shiftAmount()) {}

  uint32_t encode() const { return shiftAmount_; }

};

class RRS {

 protected:

  // Mark as a protected field to avoid unused private field warnings.

  uint32_t mustZero_ : 1;

 private:

  // The register that holds the shift amount.

  uint32_t rs_ : 4;

 public:

  explicit RRS(uint32_t rs) : rs_(rs) { MOZ_ASSERT(rs_ == rs); }

  uint32_t encode() const { return rs_ << 1; }

};

}  // namespace datastore

class MacroAssemblerARM;

class Operand;

class Operand2 {

  friend class Operand;

  friend class MacroAssemblerARM;

  friend class InstALU;

  uint32_t oper_ : 31;

  uint32_t invalid_ : 1;

 protected:

  explicit Operand2(datastore::Imm8mData base)

      : oper_(base.invalid() ? -1 : (base.encode() | uint32_t(IsImmOp2))),

        invalid_(base.invalid()) {}

  explicit Operand2(datastore::Reg base)

      : oper_(base.encode() | uint32_t(IsNotImmOp2)), invalid_(false) {}

 private:

  explicit Operand2(uint32_t blob) : oper_(blob), invalid_(false) {}

 public:

  bool isO2Reg() const { return !(oper_ & IsImmOp2); }

  Op2Reg toOp2Reg() const;

  bool isImm8() const { return oper_ & IsImmOp2; }

  bool invalid() const { return invalid_; }

  uint32_t encode() const { return oper_; }

};

class Imm8 : public Operand2 {

 public:

  explicit Imm8(uint32_t imm) : Operand2(EncodeImm(imm)) {}

  static datastore::Imm8mData EncodeImm(uint32_t imm) {

    // RotateLeft below may not be called with a shift of zero.

    if (imm <= 0xFF) {

      return datastore::Imm8mData(imm, 0);

    // An encodable integer has a maximum of 8 contiguous set bits,

    // with an optional wrapped left rotation to even bit positions.

    for (int rot = 1; rot < 16; rot++) {

      uint32_t rotimm = mozilla::RotateLeft(imm, rot * 2);

      if (rotimm <= 0xFF) {

        return datastore::Imm8mData(rotimm, rot);

    return datastore::Imm8mData();

  // Pair template?

  struct TwoImm8mData {

    datastore::Imm8mData fst_, snd_;

    TwoImm8mData() = default;

    TwoImm8mData(datastore::Imm8mData fst, datastore::Imm8mData snd)

        : fst_(fst), snd_(snd) {}

    datastore::Imm8mData fst() const { return fst_; }

    datastore::Imm8mData snd() const { return snd_; }

};

  static TwoImm8mData EncodeTwoImms(uint32_t);

};

class Op2Reg : public Operand2 {

 public:

  explicit Op2Reg(Register rm, ShiftType type, datastore::RIS shiftImm)

      : Operand2(datastore::Reg(rm.code(), type, 0, shiftImm.encode())) {}

  explicit Op2Reg(Register rm, ShiftType type, datastore::RRS shiftReg)

      : Operand2(datastore::Reg(rm.code(), type, 1, shiftReg.encode())) {}

};

static_assert(sizeof(Op2Reg) == sizeof(datastore::Reg),

              "datastore::Reg(const Op2Reg&) constructor relies on Reg/Op2Reg "

              "having same size");

class O2RegImmShift : public Op2Reg {

 public:

  explicit O2RegImmShift(Register rn, ShiftType type, uint32_t shift)

      : Op2Reg(rn, type, datastore::RIS(shift)) {}

};

class O2RegRegShift : public Op2Reg {

 public:

  explicit O2RegRegShift(Register rn, ShiftType type, Register rs)

      : Op2Reg(rn, type, datastore::RRS(rs.code())) {}

};

O2RegImmShift O2Reg(Register r);

O2RegImmShift lsl(Register r, int amt);

O2RegImmShift lsr(Register r, int amt);

O2RegImmShift asr(Register r, int amt);

O2RegImmShift rol(Register r, int amt);

O2RegImmShift ror(Register r, int amt);

O2RegRegShift lsl(Register r, Register amt);

O2RegRegShift lsr(Register r, Register amt);

O2RegRegShift asr(Register r, Register amt);

O2RegRegShift ror(Register r, Register amt);

// An offset from a register to be used for ldr/str. This should include the

// sign bit, since ARM has "signed-magnitude" offsets. That is it encodes an

// unsigned offset, then the instruction specifies if the offset is positive or

// negative. The +/- bit is necessary if the instruction set wants to be able to

// have a negative register offset e.g. ldr pc, [r1,-r2];

class DtrOff {

  uint32_t data_;

 protected:

  explicit DtrOff(datastore::Imm12Data immdata, IsUp_ iu)

      : data_(immdata.encode() | uint32_t(IsImmDTR) | uint32_t(iu)) {}

  explicit DtrOff(datastore::Reg reg, IsUp_ iu = IsUp)

      : data_(reg.encode() | uint32_t(IsNotImmDTR) | iu) {}

 public:

  uint32_t encode() const { return data_; }

};

class DtrOffImm : public DtrOff {

 public:

  explicit DtrOffImm(int32_t imm)

      : DtrOff(datastore::Imm12Data(mozilla::Abs(imm)),

               imm >= 0 ? IsUp : IsDown) {

    MOZ_ASSERT(mozilla::Abs(imm) < 4096);

};

class DtrOffReg : public DtrOff {

  // These are designed to be called by a constructor of a subclass.

  // Constructing the necessary RIS/RRS structures is annoying.

 protected:

  explicit DtrOffReg(Register rn, ShiftType type, datastore::RIS shiftImm,

                     IsUp_ iu = IsUp)

      : DtrOff(datastore::Reg(rn.code(), type, 0, shiftImm.encode()), iu) {}

  explicit DtrOffReg(Register rn, ShiftType type, datastore::RRS shiftReg,

                     IsUp_ iu = IsUp)

      : DtrOff(datastore::Reg(rn.code(), type, 1, shiftReg.encode()), iu) {}

};

class DtrRegImmShift : public DtrOffReg {

 public:

  explicit DtrRegImmShift(Register rn, ShiftType type, uint32_t shift,

                          IsUp_ iu = IsUp)

      : DtrOffReg(rn, type, datastore::RIS(shift), iu) {}

};

class DtrRegRegShift : public DtrOffReg {

 public:

  explicit DtrRegRegShift(Register rn, ShiftType type, Register rs,

                          IsUp_ iu = IsUp)

      : DtrOffReg(rn, type, datastore::RRS(rs.code()), iu) {}

};

// We will frequently want to bundle a register with its offset so that we have

// an "operand" to a load instruction.

class DTRAddr {

  friend class Operand;

  uint32_t data_;

 public:

  explicit DTRAddr(Register reg, DtrOff dtr)

      : data_(dtr.encode() | (reg.code() << 16)) {}

  uint32_t encode() const { return data_; }

  Register getBase() const { return Register::FromCode((data_ >> 16) & 0xf); }

};

// Offsets for the extended data transfer instructions:

// ldrsh, ldrd, ldrsb, etc.

class EDtrOff {

  uint32_t data_;

 protected:

  explicit EDtrOff(datastore::Imm8Data imm8, IsUp_ iu = IsUp)

      : data_(imm8.encode() | IsImmEDTR | uint32_t(iu)) {}

  explicit EDtrOff(Register rm, IsUp_ iu = IsUp)

      : data_(rm.code() | IsNotImmEDTR | iu) {}

 public:

  uint32_t encode() const { return data_; }

};

class EDtrOffImm : public EDtrOff {

 public:

  explicit EDtrOffImm(int32_t imm)

      : EDtrOff(datastore::Imm8Data(mozilla::Abs(imm)),

                (imm >= 0) ? IsUp : IsDown) {

    MOZ_ASSERT(mozilla::Abs(imm) < 256);

};

// This is the most-derived class, since the extended data transfer instructions

// don't support any sort of modifying the "index" operand.

class EDtrOffReg : public EDtrOff {

 public:

  explicit EDtrOffReg(Register rm) : EDtrOff(rm) {}

};

class EDtrAddr {

  uint32_t data_;

 public:

  explicit EDtrAddr(Register r, EDtrOff off) : data_(RN(r) | off.encode()) {}

  uint32_t encode() const { return data_; }

#ifdef DEBUG

  Register maybeOffsetRegister() const {

    if (data_ & IsImmEDTR) {

      return InvalidReg;

    return Register::FromCode(data_ & 0xf);

#endif

};

class VFPOff {

  uint32_t data_;

 protected:

  explicit VFPOff(datastore::Imm8VFPOffData imm, IsUp_ isup)

      : data_(imm.encode() | uint32_t(isup)) {}

 public:

  uint32_t encode() const { return data_; }

};

class VFPOffImm : public VFPOff {

 public:

  explicit VFPOffImm(int32_t imm)

      : VFPOff(datastore::Imm8VFPOffData(mozilla::Abs(imm) / 4),

               imm < 0 ? IsDown : IsUp) {

    MOZ_ASSERT(mozilla::Abs(imm) <= 255 * 4);

};

class VFPAddr {

  friend class Operand;

  uint32_t data_;

 public:

  explicit VFPAddr(Register base, VFPOff off)

      : data_(RN(base) | off.encode()) {}

  uint32_t encode() const { return data_; }

};

class VFPImm {

  uint32_t data_;

 public:

  explicit VFPImm(uint32_t topWordOfDouble);

  static const VFPImm One;

  uint32_t encode() const { return data_; }

  bool isValid() const { return data_ != (~0U); }

};

// A BOffImm is an immediate that is used for branches. Namely, it is the offset

// that will be encoded in the branch instruction. This is the only sane way of

// constructing a branch.

class BOffImm {

  friend class InstBranchImm;

  uint32_t data_;

 public:

  explicit BOffImm(int offset) : data_((offset - 8) >> 2 & 0x00ffffff) {

    MOZ_ASSERT((offset & 0x3) == 0);

    if (!IsInRange(offset)) {

      MOZ_CRASH("BOffImm offset out of range");

  explicit BOffImm() : data_(INVALID) {}

 private:

  explicit BOffImm(const Instruction& inst);

 public:

  static const uint32_t INVALID = 0x00800000;

  uint32_t encode() const { return data_; }

  int32_t decode() const { return ((int32_t(data_) << 8) >> 6) + 8; }

  static bool IsInRange(int offset) {

    if ((offset - 8) < -33554432) {

      return false;

    if ((offset - 8) > 33554428) {

      return false;

    return true;

  bool isInvalid() const { return data_ == INVALID; }

  Instruction* getDest(Instruction* src) const;

};

class Imm16 {

  uint32_t lower_ : 12;

 protected:

  // Mark as a protected field to avoid unused private field warnings.

  uint32_t pad_ : 4;

 private:

  uint32_t upper_ : 4;

  uint32_t invalid_ : 12;

 public:

  explicit Imm16();

  explicit Imm16(uint32_t imm);

  explicit Imm16(Instruction& inst);

  uint32_t encode() const { return lower_ | (upper_ << 16); }

  uint32_t decode() const { return lower_ | (upper_ << 12); }

  bool isInvalid() const { return invalid_; }

};

// I would preffer that these do not exist, since there are essentially no

// instructions that would ever take more than one of these, however, the MIR

// wants to only have one type of arguments to functions, so bugger.

class Operand {

  // The encoding of registers is the same for OP2, DTR and EDTR yet the type

  // system doesn't let us express this, so choices must be made.

 public:

  enum class Tag : uint8_t { OP2, MEM, FOP };

 private:

  uint32_t tag_ : 8;

  uint32_t reg_ : 5;

  int32_t offset_;

 protected:

  Operand(Tag tag, uint32_t regCode, int32_t offset)

      : tag_(static_cast<uint32_t>(tag)), reg_(regCode), offset_(offset) {}

 public:

  explicit Operand(Register reg) : Operand(Tag::OP2, reg.code(), 0) {}

  explicit Operand(FloatRegister freg) : Operand(Tag::FOP, freg.code(), 0) {}

  explicit Operand(Register base, Imm32 off)

      : Operand(Tag::MEM, base.code(), off.value) {}

  explicit Operand(Register base, int32_t off)

      : Operand(Tag::MEM, base.code(), off) {}

  explicit Operand(const Address& addr)

      : Operand(Tag::MEM, addr.base.code(), addr.offset) {}

 public:

  Tag tag() const { return static_cast<Tag>(tag_); }

  Operand2 toOp2() const {

    MOZ_ASSERT(tag() == Tag::OP2);

    return O2Reg(Register::FromCode(reg_));

  Register toReg() const {

    MOZ_ASSERT(tag() == Tag::OP2);

    return Register::FromCode(reg_);

  Address toAddress() const {

    MOZ_ASSERT(tag() == Tag::MEM);

    return Address(Register::FromCode(reg_), offset_);

  int32_t disp() const {

    MOZ_ASSERT(tag() == Tag::MEM);

    return offset_;

  int32_t base() const {

    MOZ_ASSERT(tag() == Tag::MEM);

    return reg_;

  Register baseReg() const {

    MOZ_ASSERT(tag() == Tag::MEM);

    return Register::FromCode(reg_);

  DTRAddr toDTRAddr() const {

    MOZ_ASSERT(tag() == Tag::MEM);

    return DTRAddr(baseReg(), DtrOffImm(offset_));

  VFPAddr toVFPAddr() const {

    MOZ_ASSERT(tag() == Tag::MEM);

    return VFPAddr(baseReg(), VFPOffImm(offset_));

};

inline Imm32 Imm64::firstHalf() const { return low(); }

inline Imm32 Imm64::secondHalf() const { return hi(); }

class InstructionIterator {

 private:

  Instruction* inst_;

 public:

  explicit InstructionIterator(Instruction* inst) : inst_(inst) {

    maybeSkipAutomaticInstructions();

  // Advances to the next intentionally-inserted instruction.

  Instruction* next();

  // Advances past any automatically-inserted instructions.

  Instruction* maybeSkipAutomaticInstructions();

  Instruction* cur() const { return inst_; }

 protected:

  // Advances past the given number of instruction-length bytes.

  inline void advanceRaw(ptrdiff_t instructions = 1);

};

class Assembler;

typedef js::jit::AssemblerBufferWithConstantPools<1024, 4, Instruction,

                                                  Assembler>

    ARMBuffer;

class Assembler : public AssemblerShared {

 public:

  // ARM conditional constants:

  enum ARMCondition : uint32_t {

    EQ = 0x00000000,  // Zero

    NE = 0x10000000,  // Non-zero

    CS = 0x20000000,

    CC = 0x30000000,

    MI = 0x40000000,

    PL = 0x50000000,

    VS = 0x60000000,

    VC = 0x70000000,

    HI = 0x80000000,

    LS = 0x90000000,

    GE = 0xa0000000,

    LT = 0xb0000000,

    GT = 0xc0000000,

    LE = 0xd0000000,

    AL = 0xe0000000

};

  enum Condition : uint32_t {

    Equal = EQ,

    NotEqual = NE,

    Above = HI,

    AboveOrEqual = CS,

    Below = CC,

    BelowOrEqual = LS,

    GreaterThan = GT,

    GreaterThanOrEqual = GE,

    LessThan = LT,

    LessThanOrEqual = LE,

    Overflow = VS,

    CarrySet = CS,

    CarryClear = CC,

    Signed = MI,

    NotSigned = PL,

    Zero = EQ,

    NonZero = NE,

    Always = AL,

    VFP_NotEqualOrUnordered = NE,

    VFP_Equal = EQ,

    VFP_Unordered = VS,

    VFP_NotUnordered = VC,

    VFP_GreaterThanOrEqualOrUnordered = CS,

    VFP_GreaterThanOrEqual = GE,

    VFP_GreaterThanOrUnordered = HI,

    VFP_GreaterThan = GT,

    VFP_LessThanOrEqualOrUnordered = LE,

    VFP_LessThanOrEqual = LS,

    VFP_LessThanOrUnordered = LT,

    VFP_LessThan = CC  // MI is valid too.

};

  // Bit set when a DoubleCondition does not map to a single ARM condition.

  // The macro assembler has to special-case these conditions, or else

  // ConditionFromDoubleCondition will complain.

  static const int DoubleConditionBitSpecial = 0x1;

  enum DoubleCondition : uint32_t {

    // These conditions will only evaluate to true if the comparison is

    // ordered - i.e. neither operand is NaN.

    DoubleOrdered = VFP_NotUnordered,

    DoubleEqual = VFP_Equal,

    DoubleNotEqual = VFP_NotEqualOrUnordered | DoubleConditionBitSpecial,

    DoubleGreaterThan = VFP_GreaterThan,

    DoubleGreaterThanOrEqual = VFP_GreaterThanOrEqual,

    DoubleLessThan = VFP_LessThan,

    DoubleLessThanOrEqual = VFP_LessThanOrEqual,

    // If either operand is NaN, these conditions always evaluate to true.

    DoubleUnordered = VFP_Unordered,

    DoubleEqualOrUnordered = VFP_Equal | DoubleConditionBitSpecial,

    DoubleNotEqualOrUnordered = VFP_NotEqualOrUnordered,

    DoubleGreaterThanOrUnordered = VFP_GreaterThanOrUnordered,

    DoubleGreaterThanOrEqualOrUnordered = VFP_GreaterThanOrEqualOrUnordered,

    DoubleLessThanOrUnordered = VFP_LessThanOrUnordered,

    DoubleLessThanOrEqualOrUnordered = VFP_LessThanOrEqualOrUnordered

};

  Condition getCondition(uint32_t inst) {

    return (Condition)(0xf0000000 & inst);

  static inline Condition ConditionFromDoubleCondition(DoubleCondition cond) {

    MOZ_ASSERT(!(cond & DoubleConditionBitSpecial));

    return static_cast<Condition>(cond);

  enum BarrierOption {

    BarrierSY = 15,  // Full system barrier

    BarrierST = 14   // StoreStore barrier

};

  // This should be protected, but since CodeGenerator wants to use it, it

  // needs to go out here :(

  BufferOffset nextOffset() { return m_buffer.nextOffset(); }

 protected:

  // Shim around AssemblerBufferWithConstantPools::allocEntry.

  BufferOffset allocLiteralLoadEntry(size_t numInst, unsigned numPoolEntries,

                                     PoolHintPun& php, uint8_t* data,

                                     const LiteralDoc& doc = LiteralDoc(),

                                     ARMBuffer::PoolEntry* pe = nullptr,

                                     bool loadToPC = false);

  Instruction* editSrc(BufferOffset bo) { return m_buffer.getInst(bo); }

#ifdef JS_DISASM_ARM

  typedef disasm::EmbeddedVector<char, disasm::ReasonableBufferSize>

      DisasmBuffer;

  static void disassembleInstruction(const Instruction* i,

                                     DisasmBuffer& buffer);

  void initDisassembler();

  void finishDisassembler();

  void spew(Instruction* i);

  void spewBranch(Instruction* i, const LabelDoc& target);

  void spewLiteralLoad(PoolHintPun& php, bool loadToPC, const Instruction* offs,

                       const LiteralDoc& doc);

#endif

 public:

  void resetCounter();

  static uint32_t NopFill;

  static uint32_t GetNopFill();

  static uint32_t AsmPoolMaxOffset;

  static uint32_t GetPoolMaxOffset();

 protected:

  // Structure for fixing up pc-relative loads/jumps when a the machine code

  // gets moved (executable copy, gc, etc.).

  class RelativePatch {

    void* target_;

    RelocationKind kind_;

   public:

    RelativePatch(void* target, RelocationKind kind)

        : target_(target), kind_(kind) {}

    void* target() const { return target_; }

    RelocationKind kind() const { return kind_; }

};

  // TODO: this should actually be a pool-like object. It is currently a big

  // hack, and probably shouldn't exist.

  js::Vector<RelativePatch, 8, SystemAllocPolicy> jumps_;

  CompactBufferWriter jumpRelocations_;

  CompactBufferWriter dataRelocations_;

  ARMBuffer m_buffer;

#ifdef JS_DISASM_ARM

  DisassemblerSpew spew_;

#endif

 public:

  // For the alignment fill use NOP: 0x0320f000 or (Always | InstNOP::NopInst).

  // For the nopFill use a branch to the next instruction: 0xeaffffff.

  Assembler()

      : m_buffer(1, 1, 8, GetPoolMaxOffset(), 8, 0xe320f000, 0xeaffffff,

                 GetNopFill()),

        isFinished(false),

        dtmActive(false),

        dtmCond(Always) {

#ifdef JS_DISASM_ARM

    initDisassembler();

#endif

  ~Assembler() {

#ifdef JS_DISASM_ARM

    finishDisassembler();

#endif

  void setUnlimitedBuffer() { m_buffer.setUnlimited(); }

  static Condition InvertCondition(Condition cond);

  static Condition UnsignedCondition(Condition cond);

  static Condition ConditionWithoutEqual(Condition cond);

  static DoubleCondition InvertCondition(DoubleCondition cond);

  void writeDataRelocation(BufferOffset offset, ImmGCPtr ptr) {

    // Raw GC pointer relocations and Value relocations both end up in

    // Assembler::TraceDataRelocations.

    if (ptr.value) {

      if (gc::IsInsideNursery(ptr.value)) {

        embedsNurseryPointers_ = true;

      dataRelocations_.writeUnsigned(offset.getOffset());

  enum RelocBranchStyle { B_MOVWT, B_LDR_BX, B_LDR, B_MOVW_ADD };

  enum RelocStyle { L_MOVWT, L_LDR };

 public:

  // Given the start of a Control Flow sequence, grab the value that is

  // finally branched to given the start of a function that loads an address

  // into a register get the address that ends up in the register.

  template <class Iter>

  static const uint32_t* GetCF32Target(Iter* iter);

  static uintptr_t GetPointer(uint8_t*);

  static const uint32_t* GetPtr32Target(InstructionIterator iter,

                                        Register* dest = nullptr,

                                        RelocStyle* rs = nullptr);

  bool oom() const;

  void setPrinter(Sprinter* sp) {

#ifdef JS_DISASM_ARM

    spew_.setPrinter(sp);

#endif

  Register getStackPointer() const { return StackPointer; }

 private:

  bool isFinished;

 protected:

  LabelDoc refLabel(const Label* label) {

#ifdef JS_DISASM_ARM

    return spew_.refLabel(label);

#else

    return LabelDoc();

#endif

 public:

  void finish();

  bool appendRawCode(const uint8_t* code, size_t numBytes);

  bool reserve(size_t size);

  bool swapBuffer(wasm::Bytes& bytes);

  void copyJumpRelocationTable(uint8_t* dest);

  void copyDataRelocationTable(uint8_t* dest);

  // Size of the instruction stream, in bytes, after pools are flushed.

  size_t size() const;

  // Size of the jump relocation table, in bytes.

  size_t jumpRelocationTableBytes() const;

  size_t dataRelocationTableBytes() const;

  // Size of the data table, in bytes.

  size_t bytesNeeded() const;

  // Write a single instruction into the instruction stream.  Very hot,

  // inlined for performance

  MOZ_ALWAYS_INLINE BufferOffset writeInst(uint32_t x) {

    MOZ_ASSERT(hasCreator());

    BufferOffset offs = m_buffer.putInt(x);

#ifdef JS_DISASM_ARM

    spew(m_buffer.getInstOrNull(offs));

#endif

    return offs;

  // As above, but also mark the instruction as a branch.  Very hot, inlined

  // for performance

  MOZ_ALWAYS_INLINE BufferOffset

  writeBranchInst(uint32_t x, const LabelDoc& documentation) {

    BufferOffset offs = m_buffer.putInt(x);

#ifdef JS_DISASM_ARM

    spewBranch(m_buffer.getInstOrNull(offs), documentation);

#endif

    return offs;

  // Write a placeholder NOP for a branch into the instruction stream

  // (in order to adjust assembler addresses and mark it as a branch), it will

  // be overwritten subsequently.

  BufferOffset allocBranchInst();

  // A static variant for the cases where we don't want to have an assembler

  // object.

  static void WriteInstStatic(uint32_t x, uint32_t* dest);

 public:

  void writeCodePointer(CodeLabel* label);

  void haltingAlign(int alignment);

  void nopAlign(int alignment);

  BufferOffset as_nop();

  BufferOffset as_alu(Register dest, Register src1, Operand2 op2, ALUOp op,

                      SBit s = LeaveCC, Condition c = Always);

  BufferOffset as_mov(Register dest, Operand2 op2, SBit s = LeaveCC,

                      Condition c = Always);

  BufferOffset as_mvn(Register dest, Operand2 op2, SBit s = LeaveCC,

                      Condition c = Always);

  static void as_alu_patch(Register dest, Register src1, Operand2 op2, ALUOp op,

                           SBit s, Condition c, uint32_t* pos);

  static void as_mov_patch(Register dest, Operand2 op2, SBit s, Condition c,

                           uint32_t* pos);

  // Logical operations:

  BufferOffset as_and(Register dest, Register src1, Operand2 op2,

                      SBit s = LeaveCC, Condition c = Always);

  BufferOffset as_bic(Register dest, Register src1, Operand2 op2,

                      SBit s = LeaveCC, Condition c = Always);

  BufferOffset as_eor(Register dest, Register src1, Operand2 op2,

                      SBit s = LeaveCC, Condition c = Always);

  BufferOffset as_orr(Register dest, Register src1, Operand2 op2,

                      SBit s = LeaveCC, Condition c = Always);

  // Reverse byte operations:

  BufferOffset as_rev(Register dest, Register src, Condition c = Always);

  BufferOffset as_rev16(Register dest, Register src, Condition c = Always);

  BufferOffset as_revsh(Register dest, Register src, Condition c = Always);

  // Mathematical operations:

  BufferOffset as_adc(Register dest, Register src1, Operand2 op2,

                      SBit s = LeaveCC, Condition c = Always);

  BufferOffset as_add(Register dest, Register src1, Operand2 op2,

                      SBit s = LeaveCC, Condition c = Always);

  BufferOffset as_sbc(Register dest, Register src1, Operand2 op2,

                      SBit s = LeaveCC, Condition c = Always);

  BufferOffset as_sub(Register dest, Register src1, Operand2 op2,

                      SBit s = LeaveCC, Condition c = Always);

  BufferOffset as_rsb(Register dest, Register src1, Operand2 op2,

                      SBit s = LeaveCC, Condition c = Always);

  BufferOffset as_rsc(Register dest, Register src1, Operand2 op2,

                      SBit s = LeaveCC, Condition c = Always);

  // Test operations:

  BufferOffset as_cmn(Register src1, Operand2 op2, Condition c = Always);

  BufferOffset as_cmp(Register src1, Operand2 op2, Condition c = Always);

  BufferOffset as_teq(Register src1, Operand2 op2, Condition c = Always);

  BufferOffset as_tst(Register src1, Operand2 op2, Condition c = Always);

  // Sign extension operations:

  BufferOffset as_sxtb(Register dest, Register src, int rotate,

                       Condition c = Always);

  BufferOffset as_sxth(Register dest, Register src, int rotate,

                       Condition c = Always);

  BufferOffset as_uxtb(Register dest, Register src, int rotate,

                       Condition c = Always);

  BufferOffset as_uxth(Register dest, Register src, int rotate,

                       Condition c = Always);

  // Not quite ALU worthy, but useful none the less: These also have the issue

  // of these being formatted completly differently from the standard ALU

  // operations.

  BufferOffset as_movw(Register dest, Imm16 imm, Condition c = Always);

  BufferOffset as_movt(Register dest, Imm16 imm, Condition c = Always);

  static void as_movw_patch(Register dest, Imm16 imm, Condition c,

                            Instruction* pos);

  static void as_movt_patch(Register dest, Imm16 imm, Condition c,

                            Instruction* pos);

  BufferOffset as_genmul(Register d1, Register d2, Register rm, Register rn,

                         MULOp op, SBit s, Condition c = Always);

  BufferOffset as_mul(Register dest, Register src1, Register src2,

                      SBit s = LeaveCC, Condition c = Always);

  BufferOffset as_mla(Register dest, Register acc, Register src1, Register src2,

                      SBit s = LeaveCC, Condition c = Always);

  BufferOffset as_umaal(Register dest1, Register dest2, Register src1,

                        Register src2, Condition c = Always);

  BufferOffset as_mls(Register dest, Register acc, Register src1, Register src2,

                      Condition c = Always);

  BufferOffset as_umull(Register dest1, Register dest2, Register src1,

                        Register src2, SBit s = LeaveCC, Condition c = Always);

  BufferOffset as_umlal(Register dest1, Register dest2, Register src1,

                        Register src2, SBit s = LeaveCC, Condition c = Always);

  BufferOffset as_smull(Register dest1, Register dest2, Register src1,

                        Register src2, SBit s = LeaveCC, Condition c = Always);

  BufferOffset as_smlal(Register dest1, Register dest2, Register src1,

                        Register src2, SBit s = LeaveCC, Condition c = Always);

  BufferOffset as_sdiv(Register dest, Register num, Register div,

                       Condition c = Always);

  BufferOffset as_udiv(Register dest, Register num, Register div,

                       Condition c = Always);

  BufferOffset as_clz(Register dest, Register src, Condition c = Always);

  // Data transfer instructions: ldr, str, ldrb, strb.

  // Using an int to differentiate between 8 bits and 32 bits is overkill.

  BufferOffset as_dtr(LoadStore ls, int size, Index mode, Register rt,

                      DTRAddr addr, Condition c = Always);

  static void as_dtr_patch(LoadStore ls, int size, Index mode, Register rt,

                           DTRAddr addr, Condition c, uint32_t* dest);

  // Handles all of the other integral data transferring functions:

  // ldrsb, ldrsh, ldrd, etc. The size is given in bits.

  BufferOffset as_extdtr(LoadStore ls, int size, bool IsSigned, Index mode,

                         Register rt, EDtrAddr addr, Condition c = Always);

  BufferOffset as_dtm(LoadStore ls, Register rn, uint32_t mask, DTMMode mode,

                      DTMWriteBack wb, Condition c = Always);

  // Overwrite a pool entry with new data.

  static void WritePoolEntry(Instruction* addr, Condition c, uint32_t data);

  // Load a 32 bit immediate from a pool into a register.

  BufferOffset as_Imm32Pool(Register dest, uint32_t value,

                            Condition c = Always);

  // Load a 64 bit floating point immediate from a pool into a register.

  BufferOffset as_FImm64Pool(VFPRegister dest, double value,

                             Condition c = Always);

  // Load a 32 bit floating point immediate from a pool into a register.

  BufferOffset as_FImm32Pool(VFPRegister dest, float value,

                             Condition c = Always);

  // Atomic instructions: ldrexd, ldrex, ldrexh, ldrexb, strexd, strex, strexh,

  // strexb.

//

  // The doubleword, halfword, and byte versions are available from ARMv6K

  // forward.

//

  // The word versions are available from ARMv6 forward and can be used to

  // implement the halfword and byte versions on older systems.

  // LDREXD rt, rt2, [rn].  Constraint: rt even register, rt2=rt+1.

  BufferOffset as_ldrexd(Register rt, Register rt2, Register rn,

                         Condition c = Always);

  // LDREX rt, [rn]

  BufferOffset as_ldrex(Register rt, Register rn, Condition c = Always);

  BufferOffset as_ldrexh(Register rt, Register rn, Condition c = Always);

  BufferOffset as_ldrexb(Register rt, Register rn, Condition c = Always);

  // STREXD rd, rt, rt2, [rn].  Constraint: rt even register, rt2=rt+1.

  BufferOffset as_strexd(Register rd, Register rt, Register rt2, Register rn,

                         Condition c = Always);

  // STREX rd, rt, [rn].  Constraint: rd != rn, rd != rt.

  BufferOffset as_strex(Register rd, Register rt, Register rn,

                        Condition c = Always);

  BufferOffset as_strexh(Register rd, Register rt, Register rn,

                         Condition c = Always);

  BufferOffset as_strexb(Register rd, Register rt, Register rn,

                         Condition c = Always);

  // CLREX

  BufferOffset as_clrex();

  // Memory synchronization.

  // These are available from ARMv7 forward.

  BufferOffset as_dmb(BarrierOption option = BarrierSY);

  BufferOffset as_dsb(BarrierOption option = BarrierSY);

  BufferOffset as_isb();

  // Memory synchronization for architectures before ARMv7.

  BufferOffset as_dsb_trap();

  BufferOffset as_dmb_trap();

  BufferOffset as_isb_trap();

  // Speculation barrier

  BufferOffset as_csdb();

  // Control flow stuff:

  // bx can *only* branch to a register never to an immediate.

  BufferOffset as_bx(Register r, Condition c = Always);

  // Branch can branch to an immediate *or* to a register. Branches to

  // immediates are pc relative, branches to registers are absolute.

  BufferOffset as_b(BOffImm off, Condition c, Label* documentation = nullptr);

  BufferOffset as_b(Label* l, Condition c = Always);

  BufferOffset as_b(BOffImm off, Condition c, BufferOffset inst);

  // blx can go to either an immediate or a register. When blx'ing to a

  // register, we change processor mode depending on the low bit of the

  // register when blx'ing to an immediate, we *always* change processor

  // state.

  BufferOffset as_blx(Label* l);

  BufferOffset as_blx(Register r, Condition c = Always);

  BufferOffset as_bl(BOffImm off, Condition c, Label* documentation = nullptr);

  // bl can only branch+link to an immediate, never to a register it never

  // changes processor state.

  BufferOffset as_bl();

  // bl #imm can have a condition code, blx #imm cannot.

  // blx reg can be conditional.

  BufferOffset as_bl(Label* l, Condition c);

  BufferOffset as_bl(BOffImm off, Condition c, BufferOffset inst);

  BufferOffset as_mrs(Register r, Condition c = Always);

  BufferOffset as_msr(Register r, Condition c = Always);

  // VFP instructions!

 private:

  enum vfp_size { IsDouble = 1 << 8, IsSingle = 0 << 8 };

  BufferOffset writeVFPInst(vfp_size sz, uint32_t blob);

  static void WriteVFPInstStatic(vfp_size sz, uint32_t blob, uint32_t* dest);

  // Unityped variants: all registers hold the same (ieee754 single/double)

  // notably not included are vcvt; vmov vd, #imm; vmov rt, vn.

  BufferOffset as_vfp_float(VFPRegister vd, VFPRegister vn, VFPRegister vm,

                            VFPOp op, Condition c = Always);

 public:

  BufferOffset as_vadd(VFPRegister vd, VFPRegister vn, VFPRegister vm,

                       Condition c = Always);

  BufferOffset as_vdiv(VFPRegister vd, VFPRegister vn, VFPRegister vm,

                       Condition c = Always);

  BufferOffset as_vmul(VFPRegister vd, VFPRegister vn, VFPRegister vm,

                       Condition c = Always);

  BufferOffset as_vnmul(VFPRegister vd, VFPRegister vn, VFPRegister vm,

                        Condition c = Always);

  BufferOffset as_vnmla(VFPRegister vd, VFPRegister vn, VFPRegister vm,

                        Condition c = Always);

  BufferOffset as_vnmls(VFPRegister vd, VFPRegister vn, VFPRegister vm,

                        Condition c = Always);

  BufferOffset as_vneg(VFPRegister vd, VFPRegister vm, Condition c = Always);

  BufferOffset as_vsqrt(VFPRegister vd, VFPRegister vm, Condition c = Always);

  BufferOffset as_vabs(VFPRegister vd, VFPRegister vm, Condition c = Always);

  BufferOffset as_vsub(VFPRegister vd, VFPRegister vn, VFPRegister vm,

                       Condition c = Always);

  BufferOffset as_vcmp(VFPRegister vd, VFPRegister vm, Condition c = Always);

  BufferOffset as_vcmpz(VFPRegister vd, Condition c = Always);

  // Specifically, a move between two same sized-registers.

  BufferOffset as_vmov(VFPRegister vd, VFPRegister vsrc, Condition c = Always);

  // Transfer between Core and VFP.

  enum FloatToCore_ { FloatToCore = 1 << 20, CoreToFloat = 0 << 20 };

 private:

  enum VFPXferSize { WordTransfer = 0x02000010, DoubleTransfer = 0x00400010 };

 public:

  // Unlike the next function, moving between the core registers and vfp

  // registers can't be *that* properly typed. Namely, since I don't want to

  // munge the type VFPRegister to also include core registers. Thus, the core

  // and vfp registers are passed in based on their type, and src/dest is

  // determined by the float2core.

  BufferOffset as_vxfer(Register vt1, Register vt2, VFPRegister vm,

                        FloatToCore_ f2c, Condition c = Always, int idx = 0);

  // Our encoding actually allows just the src and the dest (and their types)

  // to uniquely specify the encoding that we are going to use.

  BufferOffset as_vcvt(VFPRegister vd, VFPRegister vm, bool useFPSCR = false,

                       Condition c = Always);