firefox-main/js/src/jit/MacroAssembler-inl.h (file symbol)

Source code

Revision control

Copy as Markdown

Other Tools

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: set ts=8 sts=2 et sw=2 tw=80:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef jit_MacroAssembler_inl_h
#define jit_MacroAssembler_inl_h
#include "jit/MacroAssembler.h"
#include "mozilla/FloatingPoint.h"
#include "mozilla/MathAlgorithms.h"
#include "gc/Zone.h"
#include "jit/CalleeToken.h"
#include "jit/CompileWrappers.h"
#include "jit/JitFrames.h"
#include "jit/JSJitFrameIter.h"
#include "js/Prefs.h"
#include "vm/BigIntType.h"
#include "vm/JSObject.h"
#include "vm/ProxyObject.h"
#include "vm/Runtime.h"
#include "vm/StringType.h"
#include "jit/ABIFunctionList-inl.h"
#if defined(JS_CODEGEN_X86)
# include "jit/x86/MacroAssembler-x86-inl.h"
#elif defined(JS_CODEGEN_X64)
# include "jit/x64/MacroAssembler-x64-inl.h"
#elif defined(JS_CODEGEN_ARM)
# include "jit/arm/MacroAssembler-arm-inl.h"
#elif defined(JS_CODEGEN_ARM64)
# include "jit/arm64/MacroAssembler-arm64-inl.h"
#elif defined(JS_CODEGEN_MIPS64)
# include "jit/mips64/MacroAssembler-mips64-inl.h"
#elif defined(JS_CODEGEN_LOONG64)
# include "jit/loong64/MacroAssembler-loong64-inl.h"
#elif defined(JS_CODEGEN_RISCV64)
# include "jit/riscv64/MacroAssembler-riscv64-inl.h"
#elif defined(JS_CODEGEN_WASM32)
# include "jit/wasm32/MacroAssembler-wasm32-inl.h"
#elif !defined(JS_CODEGEN_NONE)
# error "Unknown architecture!"
#endif
#include "wasm/WasmBuiltins.h"
namespace js {
namespace jit {
template <typename Sig>
DynFn DynamicFunction(Sig fun) {
ABIFunctionSignature<Sig> sig;
return DynFn{sig.address(fun)};
}
// Helper for generatePreBarrier.
inline DynFn JitPreWriteBarrier(MIRType type) {
switch (type) {
case MIRType::Value: {
using Fn = void (*)(JSRuntime* rt, Value* vp);
return DynamicFunction<Fn>(JitValuePreWriteBarrier);
}
case MIRType::String: {
using Fn = void (*)(JSRuntime* rt, JSString** stringp);
return DynamicFunction<Fn>(JitStringPreWriteBarrier);
}
case MIRType::Object: {
using Fn = void (*)(JSRuntime* rt, JSObject** objp);
return DynamicFunction<Fn>(JitObjectPreWriteBarrier);
}
case MIRType::Shape: {
using Fn = void (*)(JSRuntime* rt, Shape** shapep);
return DynamicFunction<Fn>(JitShapePreWriteBarrier);
}
case MIRType::WasmAnyRef: {
using Fn = void (*)(JSRuntime* rt, wasm::AnyRef* refp);
return DynamicFunction<Fn>(JitWasmAnyRefPreWriteBarrier);
}
default:
MOZ_CRASH();
}
}
//{{{ check_macroassembler_style
// ===============================================================
// Stack manipulation functions.
CodeOffset MacroAssembler::PushWithPatch(ImmWord word) {
framePushed_ += sizeof(word.value);
return pushWithPatch(word);
}
CodeOffset MacroAssembler::PushWithPatch(ImmPtr imm) {
return PushWithPatch(ImmWord(uintptr_t(imm.value)));
}
// ===============================================================
// Simple call functions.
void MacroAssembler::call(TrampolinePtr code) { call(ImmPtr(code.value)); }
CodeOffset MacroAssembler::call(const wasm::CallSiteDesc& desc,
const Register reg) {
CodeOffset l = call(reg);
append(desc, l);
return l;
}
CodeOffset MacroAssembler::call(const wasm::CallSiteDesc& desc,
uint32_t funcIndex) {
CodeOffset l = callWithPatch();
append(desc, l, funcIndex);
return l;
}
void MacroAssembler::call(const wasm::CallSiteDesc& desc, wasm::Trap trap) {
CodeOffset l = callWithPatch();
append(desc, l, trap);
}
CodeOffset MacroAssembler::call(const wasm::CallSiteDesc& desc,
wasm::SymbolicAddress imm) {
CodeOffset raOffset = call(imm);
append(desc, raOffset);
return raOffset;
}
// ===============================================================
// ABI function calls.
void MacroAssembler::passABIArg(Register reg) {
passABIArg(MoveOperand(reg), ABIType::General);
}
void MacroAssembler::passABIArg(FloatRegister reg, ABIType type) {
passABIArg(MoveOperand(reg), type);
}
void MacroAssembler::callWithABI(DynFn fun, ABIType result,
CheckUnsafeCallWithABI check) {
AutoProfilerCallInstrumentation profiler(*this);
callWithABINoProfiler(fun.address, result, check);
}
template <typename Sig, Sig fun>
void MacroAssembler::callWithABI(ABIType result, CheckUnsafeCallWithABI check) {
ABIFunction<Sig, fun> abiFun;
AutoProfilerCallInstrumentation profiler(*this);
callWithABINoProfiler(abiFun.address(), result, check);
}
void MacroAssembler::callWithABI(Register fun, ABIType result) {
AutoProfilerCallInstrumentation profiler(*this);
callWithABINoProfiler(fun, result);
}
void MacroAssembler::callWithABI(const Address& fun, ABIType result) {
AutoProfilerCallInstrumentation profiler(*this);
callWithABINoProfiler(fun, result);
}
void MacroAssembler::appendSignatureType(ABIType type) {
#ifdef JS_SIMULATOR
signature_ <<= ABITypeArgShift;
signature_ |= uint32_t(type);
#endif
}
ABIFunctionType MacroAssembler::signature() const {
#ifdef JS_SIMULATOR
# ifdef DEBUG
switch (signature_) {
case Args_General0:
case Args_General1:
case Args_General2:
case Args_General3:
case Args_General4:
case Args_General5:
case Args_General6:
case Args_General7:
case Args_General8:
case Args_Double_None:
case Args_Int_Double:
case Args_Float32_Float32:
case Args_Float32_Float64:
case Args_Float32_General:
case Args_Float32_Int32:
case Args_Int_Float32:
case Args_Int32_Float32:
case Args_Double_Double:
case Args_Double_Int:
case Args_Double_DoubleInt:
case Args_Double_DoubleDouble:
case Args_Double_IntDouble:
case Args_Int_IntDouble:
case Args_Int_DoubleInt:
case Args_Int_DoubleIntInt:
case Args_Int_IntDoubleIntInt:
case Args_Double_DoubleDoubleDouble:
case Args_Double_DoubleDoubleDoubleDouble:
case Args_Int64_GeneralGeneral:
case Args_General_GeneralInt64GeneralGeneral:
case Args_General_GeneralFloat32GeneralGeneral:
break;
default:
MOZ_CRASH("Unexpected type");
}
# endif // DEBUG
return ABIFunctionType(signature_);
#else
// No simulator enabled.
MOZ_CRASH("Only available for making calls within a simulator.");
#endif
}
// ===============================================================
// Jit Frames.
uint32_t MacroAssembler::callJitNoProfiler(Register callee) {
#ifdef JS_USE_LINK_REGISTER
// The return address is pushed by the callee.
call(callee);
#else
callAndPushReturnAddress(callee);
#endif
return currentOffset();
}
uint32_t MacroAssembler::callJit(Register callee) {
AutoProfilerCallInstrumentation profiler(*this);
uint32_t ret = callJitNoProfiler(callee);
return ret;
}
uint32_t MacroAssembler::callJit(JitCode* callee) {
AutoProfilerCallInstrumentation profiler(*this);
call(callee);
return currentOffset();
}
uint32_t MacroAssembler::callJit(TrampolinePtr code) {
AutoProfilerCallInstrumentation profiler(*this);
call(code);
return currentOffset();
}
uint32_t MacroAssembler::callJit(ImmPtr callee) {
AutoProfilerCallInstrumentation profiler(*this);
call(callee);
return currentOffset();
}
void MacroAssembler::push(FrameDescriptor descriptor) {
push(Imm32(descriptor.value()));
}
void MacroAssembler::Push(FrameDescriptor descriptor) {
Push(Imm32(descriptor.value()));
}
void MacroAssembler::makeFrameDescriptorForJitCall(FrameType type,
Register argc, Register dest,
bool hasInlineICScript) {
lshift32(Imm32(FrameDescriptor::NumActualArgsShift), argc, dest);
FrameDescriptor base(type, 0, hasInlineICScript);
if (base.value()) {
or32(Imm32(base.value()), dest);
}
}
void MacroAssembler::pushFrameDescriptorForJitCall(FrameType type,
Register argc,
Register scratch,
bool hasInlineICScript) {
makeFrameDescriptorForJitCall(type, argc, scratch, hasInlineICScript);
push(scratch);
}
void MacroAssembler::PushFrameDescriptorForJitCall(FrameType type,
Register argc,
Register scratch,
bool hasInlineICScript) {
pushFrameDescriptorForJitCall(type, argc, scratch, hasInlineICScript);
framePushed_ += sizeof(uintptr_t);
}
void MacroAssembler::loadNumActualArgs(Register framePtr, Register dest) {
loadPtr(Address(framePtr, JitFrameLayout::offsetOfDescriptor()), dest);
rshift32(Imm32(FrameDescriptor::NumActualArgsShift), dest);
}
void MacroAssembler::PushCalleeToken(Register callee, bool constructing) {
if (constructing) {
orPtr(Imm32(CalleeToken_FunctionConstructing), callee);
Push(callee);
andPtr(Imm32(uint32_t(CalleeTokenMask)), callee);
} else {
static_assert(CalleeToken_Function == 0,
"Non-constructing call requires no tagging");
Push(callee);
}
}
void MacroAssembler::loadFunctionFromCalleeToken(Address token, Register dest) {
#ifdef DEBUG
Label ok;
loadPtr(token, dest);
andPtr(Imm32(uint32_t(~CalleeTokenMask)), dest);
branchPtr(Assembler::Equal, dest, Imm32(CalleeToken_Function), &ok);
branchPtr(Assembler::Equal, dest, Imm32(CalleeToken_FunctionConstructing),
&ok);
assumeUnreachable("Unexpected CalleeToken tag");
bind(&ok);
#endif
loadPtr(token, dest);
andPtr(Imm32(uint32_t(CalleeTokenMask)), dest);
}
uint32_t MacroAssembler::buildFakeExitFrame(Register scratch) {
mozilla::DebugOnly<uint32_t> initialDepth = framePushed();
Push(FrameDescriptor(FrameType::IonJS));
uint32_t retAddr = pushFakeReturnAddress(scratch);
Push(FramePointer);
MOZ_ASSERT(framePushed() == initialDepth + ExitFrameLayout::Size());
return retAddr;
}
// ===============================================================
// Exit frame footer.
void MacroAssembler::enterExitFrame(Register cxreg, Register scratch,
VMFunctionId f) {
linkExitFrame(cxreg, scratch);
// Push `ExitFrameType::VMFunction + VMFunctionId`, for marking the arguments.
// See ExitFooterFrame::data_.
uintptr_t type = uintptr_t(ExitFrameType::VMFunction) + uintptr_t(f);
MOZ_ASSERT(type <= INT32_MAX);
Push(Imm32(type));
}
void MacroAssembler::enterFakeExitFrame(Register cxreg, Register scratch,
ExitFrameType type) {
linkExitFrame(cxreg, scratch);
Push(Imm32(int32_t(type)));
}
void MacroAssembler::enterFakeExitFrameForNative(Register cxreg,
Register scratch,
bool isConstructing) {
enterFakeExitFrame(cxreg, scratch,
isConstructing ? ExitFrameType::ConstructNative
: ExitFrameType::CallNative);
}
void MacroAssembler::leaveExitFrame(size_t extraFrame) {
freeStack(ExitFooterFrame::Size() + extraFrame);
}
// ===============================================================
// Move instructions
void MacroAssembler::moveValue(const ConstantOrRegister& src,
const ValueOperand& dest) {
if (src.constant()) {
moveValue(src.value(), dest);
return;
}
moveValue(src.reg(), dest);
}
// ===============================================================
// Copy instructions
void MacroAssembler::copy64(const Address& src, const Address& dest,
Register scratch) {
#if JS_BITS_PER_WORD == 32
MOZ_RELEASE_ASSERT(src.base != scratch && dest.base != scratch);
load32(LowWord(src), scratch);
store32(scratch, LowWord(dest));
load32(HighWord(src), scratch);
store32(scratch, HighWord(dest));
#else
Register64 scratch64(scratch);
load64(src, scratch64);
store64(scratch64, dest);
#endif
}
// ===============================================================
// Arithmetic functions
void MacroAssembler::addPtr(ImmPtr imm, Register dest) {
addPtr(ImmWord(uintptr_t(imm.value)), dest);
}
// ===============================================================
// Branch functions
void MacroAssembler::branchTest64(Condition cond, Register64 lhs,
Register64 rhs, Label* success, Label* fail) {
branchTest64(cond, lhs, rhs, InvalidReg, success, fail);
}
void MacroAssembler::branchIfFalseBool(Register reg, Label* label) {
// Note that C++ bool is only 1 byte, so ignore the higher-order bits.
branchTest32(Assembler::Zero, reg, Imm32(0xFF), label);
}
void MacroAssembler::branchIfTrueBool(Register reg, Label* label) {
// Note that C++ bool is only 1 byte, so ignore the higher-order bits.
branchTest32(Assembler::NonZero, reg, Imm32(0xFF), label);
}
void MacroAssembler::branchIfNotNullOrUndefined(ValueOperand val,
Label* label) {
Label nullOrUndefined;
ScratchTagScope tag(*this, val);
splitTagForTest(val, tag);
branchTestNull(Assembler::Equal, tag, &nullOrUndefined);
branchTestUndefined(Assembler::NotEqual, tag, label);
bind(&nullOrUndefined);
}
void MacroAssembler::branchIfRope(Register str, Label* label) {
Address flags(str, JSString::offsetOfFlags());
branchTest32(Assembler::Zero, flags, Imm32(JSString::LINEAR_BIT), label);
}
void MacroAssembler::branchIfNotRope(Register str, Label* label) {
Address flags(str, JSString::offsetOfFlags());
branchTest32(Assembler::NonZero, flags, Imm32(JSString::LINEAR_BIT), label);
}
void MacroAssembler::branchLatin1String(Register string, Label* label) {
branchTest32(Assembler::NonZero, Address(string, JSString::offsetOfFlags()),
Imm32(JSString::LATIN1_CHARS_BIT), label);
}
void MacroAssembler::branchTwoByteString(Register string, Label* label) {
branchTest32(Assembler::Zero, Address(string, JSString::offsetOfFlags()),
Imm32(JSString::LATIN1_CHARS_BIT), label);
}
void MacroAssembler::branchIfBigIntIsNegative(Register bigInt, Label* label) {
branchTest32(Assembler::NonZero, Address(bigInt, BigInt::offsetOfFlags()),
Imm32(BigInt::signBitMask()), label);
}
void MacroAssembler::branchIfBigIntIsNonNegative(Register bigInt,
Label* label) {
branchTest32(Assembler::Zero, Address(bigInt, BigInt::offsetOfFlags()),
Imm32(BigInt::signBitMask()), label);
}
void MacroAssembler::branchIfBigIntIsZero(Register bigInt, Label* label) {
branch32(Assembler::Equal, Address(bigInt, BigInt::offsetOfLength()),
Imm32(0), label);
}
void MacroAssembler::branchIfBigIntIsNonZero(Register bigInt, Label* label) {
branch32(Assembler::NotEqual, Address(bigInt, BigInt::offsetOfLength()),
Imm32(0), label);
}
void MacroAssembler::branchTestFunctionFlags(Register fun, uint32_t flags,
Condition cond, Label* label) {
Address address(fun, JSFunction::offsetOfFlagsAndArgCount());
branchTest32(cond, address, Imm32(flags), label);
}
void MacroAssembler::branchIfNotFunctionIsNonBuiltinCtor(Register fun,
Register scratch,
Label* label) {
// Guard the function has the BASESCRIPT and CONSTRUCTOR flags and does NOT
// have the SELF_HOSTED flag.
// This is equivalent to JSFunction::isNonBuiltinConstructor.
constexpr int32_t mask = FunctionFlags::BASESCRIPT |
FunctionFlags::SELF_HOSTED |
FunctionFlags::CONSTRUCTOR;
constexpr int32_t expected =
FunctionFlags::BASESCRIPT | FunctionFlags::CONSTRUCTOR;
load32(Address(fun, JSFunction::offsetOfFlagsAndArgCount()), scratch);
and32(Imm32(mask), scratch);
branch32(Assembler::NotEqual, scratch, Imm32(expected), label);
}
void MacroAssembler::branchIfFunctionHasNoJitEntry(Register fun, Label* label) {
uint16_t flags = FunctionFlags::HasJitEntryFlags();
branchTestFunctionFlags(fun, flags, Assembler::Zero, label);
}
void MacroAssembler::branchIfFunctionHasJitEntry(Register fun, Label* label) {
uint16_t flags = FunctionFlags::HasJitEntryFlags();
branchTestFunctionFlags(fun, flags, Assembler::NonZero, label);
}
void MacroAssembler::branchIfScriptHasJitScript(Register script, Label* label) {
static_assert(ScriptWarmUpData::JitScriptTag == 0,
"Code below depends on tag value");
branchTestPtr(Assembler::Zero,
Address(script, JSScript::offsetOfWarmUpData()),
Imm32(ScriptWarmUpData::TagMask), label);
}
void MacroAssembler::branchIfScriptHasNoJitScript(Register script,
Label* label) {
static_assert(ScriptWarmUpData::JitScriptTag == 0,
"Code below depends on tag value");
static_assert(BaseScript::offsetOfWarmUpData() ==
SelfHostedLazyScript::offsetOfWarmUpData(),
"SelfHostedLazyScript and BaseScript must use same layout for "
"warmUpData_");
branchTestPtr(Assembler::NonZero,
Address(script, JSScript::offsetOfWarmUpData()),
Imm32(ScriptWarmUpData::TagMask), label);
}
void MacroAssembler::loadJitScript(Register script, Register dest) {
#ifdef DEBUG
Label ok;
branchIfScriptHasJitScript(script, &ok);
assumeUnreachable("Script has no JitScript!");
bind(&ok);
#endif
static_assert(ScriptWarmUpData::JitScriptTag == 0,
"Code below depends on tag value");
loadPtr(Address(script, JSScript::offsetOfWarmUpData()), dest);
}
void MacroAssembler::loadFunctionArgCount(Register func, Register output) {
load32(Address(func, JSFunction::offsetOfFlagsAndArgCount()), output);
rshift32(Imm32(JSFunction::ArgCountShift), output);
}
void MacroAssembler::branchIfObjectEmulatesUndefined(Register objReg,
Register scratch,
Label* slowCheck,
Label* label) {
MOZ_ASSERT(objReg != scratch);
Label done;
loadRuntimeFuse(RuntimeFuses::FuseIndex::HasSeenObjectEmulateUndefinedFuse,
scratch);
branchPtr(Assembler::Equal, scratch, ImmPtr(nullptr), &done);
loadObjClassUnsafe(objReg, scratch);
Address flags(scratch, JSClass::offsetOfFlags());
branchTest32(Assembler::NonZero, flags, Imm32(JSCLASS_EMULATES_UNDEFINED),
label);
// Call into C++ if the object is a wrapper.
branchTestClassIsProxy(false, scratch, &done);
branchTestProxyHandlerFamily(Assembler::Equal, objReg, scratch,
&Wrapper::family, slowCheck);
bind(&done);
}
void MacroAssembler::branchFunctionKind(Condition cond,
FunctionFlags::FunctionKind kind,
Register fun, Register scratch,
Label* label) {
Address address(fun, JSFunction::offsetOfFlagsAndArgCount());
load32(address, scratch);
and32(Imm32(FunctionFlags::FUNCTION_KIND_MASK), scratch);
branch32(cond, scratch, Imm32(kind), label);
}
void MacroAssembler::branchTestObjClass(Condition cond, Register obj,
const JSClass* clasp, Register scratch,
Register spectreRegToZero,
Label* label) {
MOZ_ASSERT(obj != scratch);
MOZ_ASSERT(scratch != spectreRegToZero);
loadPtr(Address(obj, JSObject::offsetOfShape()), scratch);
loadPtr(Address(scratch, Shape::offsetOfBaseShape()), scratch);
branchPtr(cond, Address(scratch, BaseShape::offsetOfClasp()), ImmPtr(clasp),
label);
if (JitOptions.spectreObjectMitigations) {
spectreZeroRegister(cond, scratch, spectreRegToZero);
}
}
void MacroAssembler::branchTestObjClassNoSpectreMitigations(
Condition cond, Register obj, const JSClass* clasp, Register scratch,
Label* label) {
loadPtr(Address(obj, JSObject::offsetOfShape()), scratch);
loadPtr(Address(scratch, Shape::offsetOfBaseShape()), scratch);
branchPtr(cond, Address(scratch, BaseShape::offsetOfClasp()), ImmPtr(clasp),
label);
}
void MacroAssembler::branchTestObjClass(Condition cond, Register obj,
const Address& clasp, Register scratch,
Register spectreRegToZero,
Label* label) {
MOZ_ASSERT(obj != scratch);
MOZ_ASSERT(scratch != spectreRegToZero);
loadObjClassUnsafe(obj, scratch);
branchPtr(cond, clasp, scratch, label);
if (JitOptions.spectreObjectMitigations) {
spectreZeroRegister(cond, scratch, spectreRegToZero);
}
}
void MacroAssembler::branchTestObjClassNoSpectreMitigations(
Condition cond, Register obj, const Address& clasp, Register scratch,
Label* label) {
MOZ_ASSERT(obj != scratch);
loadObjClassUnsafe(obj, scratch);
branchPtr(cond, clasp, scratch, label);
}
void MacroAssembler::branchTestObjClass(Condition cond, Register obj,
Register clasp, Register scratch,
Register spectreRegToZero,
Label* label) {
MOZ_ASSERT(obj != scratch);
MOZ_ASSERT(scratch != spectreRegToZero);
loadObjClassUnsafe(obj, scratch);
branchPtr(cond, clasp, scratch, label);
if (JitOptions.spectreObjectMitigations) {
spectreZeroRegister(cond, scratch, spectreRegToZero);
}
}
void MacroAssembler::branchTestClassIsFunction(Condition cond, Register clasp,
Label* label) {
MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
if (cond == Assembler::Equal) {
branchPtr(Assembler::Equal, clasp, ImmPtr(&FunctionClass), label);
branchPtr(Assembler::Equal, clasp, ImmPtr(&ExtendedFunctionClass), label);
return;
}
Label isClass;
branchPtr(Assembler::Equal, clasp, ImmPtr(&FunctionClass), &isClass);
branchPtr(Assembler::NotEqual, clasp, ImmPtr(&ExtendedFunctionClass), label);
bind(&isClass);
}
void MacroAssembler::branchTestObjIsFunction(Condition cond, Register obj,
Register scratch,
Register spectreRegToZero,
Label* label) {
MOZ_ASSERT(scratch != spectreRegToZero);
branchTestObjIsFunctionNoSpectreMitigations(cond, obj, scratch, label);
if (JitOptions.spectreObjectMitigations) {
spectreZeroRegister(cond, scratch, spectreRegToZero);
}
}
void MacroAssembler::branchTestObjIsFunctionNoSpectreMitigations(
Condition cond, Register obj, Register scratch, Label* label) {
MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
MOZ_ASSERT(obj != scratch);
loadObjClassUnsafe(obj, scratch);
branchTestClassIsFunction(cond, scratch, label);
}
void MacroAssembler::branchTestObjShape(Condition cond, Register obj,
const Shape* shape, Register scratch,
Register spectreRegToZero,
Label* label) {
MOZ_ASSERT(obj != scratch);
MOZ_ASSERT(spectreRegToZero != scratch);
if (JitOptions.spectreObjectMitigations) {
move32(Imm32(0), scratch);
}
branchPtr(cond, Address(obj, JSObject::offsetOfShape()), ImmGCPtr(shape),
label);
if (JitOptions.spectreObjectMitigations) {
spectreMovePtr(cond, scratch, spectreRegToZero);
}
}
void MacroAssembler::branchTestObjShapeNoSpectreMitigations(Condition cond,
Register obj,
const Shape* shape,
Label* label) {
branchPtr(cond, Address(obj, JSObject::offsetOfShape()), ImmGCPtr(shape),
label);
}
void MacroAssembler::branchTestObjShape(Condition cond, Register obj,
Register shape, Register scratch,
Register spectreRegToZero,
Label* label) {
MOZ_ASSERT(obj != scratch);
MOZ_ASSERT(obj != shape);
MOZ_ASSERT(spectreRegToZero != scratch);
if (JitOptions.spectreObjectMitigations) {
move32(Imm32(0), scratch);
}
branchPtr(cond, Address(obj, JSObject::offsetOfShape()), shape, label);
if (JitOptions.spectreObjectMitigations) {
spectreMovePtr(cond, scratch, spectreRegToZero);
}
}
void MacroAssembler::branchTestObjShapeNoSpectreMitigations(Condition cond,
Register obj,
Register shape,
Label* label) {
branchPtr(cond, Address(obj, JSObject::offsetOfShape()), shape, label);
}
void MacroAssembler::branchTestObjShapeUnsafe(Condition cond, Register obj,
Register shape, Label* label) {
branchTestObjShapeNoSpectreMitigations(cond, obj, shape, label);
}
void MacroAssembler::branchTestClassIsProxy(bool proxy, Register clasp,
Label* label) {
branchTest32(proxy ? Assembler::NonZero : Assembler::Zero,
Address(clasp, JSClass::offsetOfFlags()),
Imm32(JSCLASS_IS_PROXY), label);
}
void MacroAssembler::branchTestObjectIsProxy(bool proxy, Register object,
Register scratch, Label* label) {
constexpr uint32_t ShiftedMask = (Shape::kindMask() << Shape::kindShift());
static_assert(uint32_t(Shape::Kind::Proxy) == 0,
"branchTest32 below depends on proxy kind being 0");
loadPtr(Address(object, JSObject::offsetOfShape()), scratch);
branchTest32(proxy ? Assembler::Zero : Assembler::NonZero,
Address(scratch, Shape::offsetOfImmutableFlags()),
Imm32(ShiftedMask), label);
}
void MacroAssembler::branchTestProxyHandlerFamily(Condition cond,
Register proxy,
Register scratch,
const void* handlerp,
Label* label) {
#ifdef DEBUG
Label ok;
branchTestObjectIsProxy(true, proxy, scratch, &ok);
assumeUnreachable("Expected ProxyObject in branchTestProxyHandlerFamily");
bind(&ok);
#endif
Address handlerAddr(proxy, ProxyObject::offsetOfHandler());
loadPtr(handlerAddr, scratch);
Address familyAddr(scratch, BaseProxyHandler::offsetOfFamily());
branchPtr(cond, familyAddr, ImmPtr(handlerp), label);
}
void MacroAssembler::branchTestNeedsIncrementalBarrier(Condition cond,
Label* label) {
MOZ_ASSERT(cond == Zero || cond == NonZero);
CompileZone* zone = realm()->zone();
const uint32_t* needsBarrierAddr = zone->addressOfNeedsIncrementalBarrier();
branchTest32(cond, AbsoluteAddress(needsBarrierAddr), Imm32(0x1), label);
}
void MacroAssembler::branchTestNeedsIncrementalBarrierAnyZone(
Condition cond, Label* label, Register scratch) {
MOZ_ASSERT(cond == Zero || cond == NonZero);
if (maybeRealm_) {
branchTestNeedsIncrementalBarrier(cond, label);
} else {
// We are compiling the interpreter or another runtime-wide trampoline, so
// we have to load cx->zone.
loadPtr(AbsoluteAddress(runtime()->addressOfZone()), scratch);
Address needsBarrierAddr(scratch, Zone::offsetOfNeedsIncrementalBarrier());
branchTest32(cond, needsBarrierAddr, Imm32(0x1), label);
}
}
void MacroAssembler::branchTestMagicValue(Condition cond,
const ValueOperand& val,
JSWhyMagic why, Label* label) {
MOZ_ASSERT(cond == Equal || cond == NotEqual);
branchTestValue(cond, val, MagicValue(why), label);
}
void MacroAssembler::branchDoubleNotInInt64Range(Address src, Register temp,
Label* fail) {
using mozilla::FloatingPoint;
// Tests if double is in [INT64_MIN; INT64_MAX] range
uint32_t EXPONENT_MASK = 0x7ff00000;
uint32_t EXPONENT_SHIFT = FloatingPoint<double>::kExponentShift - 32;
uint32_t TOO_BIG_EXPONENT = (FloatingPoint<double>::kExponentBias + 63)
<< EXPONENT_SHIFT;
load32(Address(src.base, src.offset + sizeof(int32_t)), temp);
and32(Imm32(EXPONENT_MASK), temp);
branch32(Assembler::GreaterThanOrEqual, temp, Imm32(TOO_BIG_EXPONENT), fail);
}
void MacroAssembler::branchDoubleNotInUInt64Range(Address src, Register temp,
Label* fail) {
using mozilla::FloatingPoint;
// Note: returns failure on -0.0
// Tests if double is in [0; UINT64_MAX] range
// Take the sign also in the equation. That way we can compare in one test?
uint32_t EXPONENT_MASK = 0xfff00000;
uint32_t EXPONENT_SHIFT = FloatingPoint<double>::kExponentShift - 32;
uint32_t TOO_BIG_EXPONENT = (FloatingPoint<double>::kExponentBias + 64)
<< EXPONENT_SHIFT;
load32(Address(src.base, src.offset + sizeof(int32_t)), temp);
and32(Imm32(EXPONENT_MASK), temp);
branch32(Assembler::AboveOrEqual, temp, Imm32(TOO_BIG_EXPONENT), fail);
}
void MacroAssembler::branchFloat32NotInInt64Range(Address src, Register temp,
Label* fail) {
using mozilla::FloatingPoint;
// Tests if float is in [INT64_MIN; INT64_MAX] range
uint32_t EXPONENT_MASK = 0x7f800000;
uint32_t EXPONENT_SHIFT = FloatingPoint<float>::kExponentShift;
uint32_t TOO_BIG_EXPONENT = (FloatingPoint<float>::kExponentBias + 63)
<< EXPONENT_SHIFT;
load32(src, temp);
and32(Imm32(EXPONENT_MASK), temp);
branch32(Assembler::GreaterThanOrEqual, temp, Imm32(TOO_BIG_EXPONENT), fail);
}
void MacroAssembler::branchFloat32NotInUInt64Range(Address src, Register temp,
Label* fail) {
using mozilla::FloatingPoint;
// Note: returns failure on -0.0
// Tests if float is in [0; UINT64_MAX] range
// Take the sign also in the equation. That way we can compare in one test?
uint32_t EXPONENT_MASK = 0xff800000;
uint32_t EXPONENT_SHIFT = FloatingPoint<float>::kExponentShift;
uint32_t TOO_BIG_EXPONENT = (FloatingPoint<float>::kExponentBias + 64)
<< EXPONENT_SHIFT;
load32(src, temp);
and32(Imm32(EXPONENT_MASK), temp);
branch32(Assembler::AboveOrEqual, temp, Imm32(TOO_BIG_EXPONENT), fail);
}
// ========================================================================
// Canonicalization primitives.
void MacroAssembler::canonicalizeFloat(FloatRegister reg) {
Label notNaN;
branchFloat(DoubleOrdered, reg, reg, &notNaN);
loadConstantFloat32(float(JS::GenericNaN()), reg);
bind(&notNaN);
}
void MacroAssembler::canonicalizeDouble(FloatRegister reg) {
Label notNaN;
branchDouble(DoubleOrdered, reg, reg, &notNaN);
loadConstantDouble(JS::GenericNaN(), reg);
bind(&notNaN);
}
// ========================================================================
// Memory access primitives.
template <class T>
void MacroAssembler::boxDouble(FloatRegister src, const T& dest) {
storeDouble(src, dest);
}
template <typename T>
void MacroAssembler::fallibleUnboxInt32(const T& src, Register dest,
Label* fail) {
// Int32Value can be unboxed efficiently with unboxInt32, so use that.
branchTestInt32(Assembler::NotEqual, src, fail);
unboxInt32(src, dest);
}
template <typename T>
void MacroAssembler::fallibleUnboxBoolean(const T& src, Register dest,
Label* fail) {
// BooleanValue can be unboxed efficiently with unboxBoolean, so use that.
branchTestBoolean(Assembler::NotEqual, src, fail);
unboxBoolean(src, dest);
}
template <typename T>
void MacroAssembler::fallibleUnboxObject(const T& src, Register dest,
Label* fail) {
fallibleUnboxPtr(src, dest, JSVAL_TYPE_OBJECT, fail);
}
template <typename T>
void MacroAssembler::fallibleUnboxString(const T& src, Register dest,
Label* fail) {
fallibleUnboxPtr(src, dest, JSVAL_TYPE_STRING, fail);
}
template <typename T>
void MacroAssembler::fallibleUnboxSymbol(const T& src, Register dest,
Label* fail) {
fallibleUnboxPtr(src, dest, JSVAL_TYPE_SYMBOL, fail);
}
template <typename T>
void MacroAssembler::fallibleUnboxBigInt(const T& src, Register dest,
Label* fail) {
fallibleUnboxPtr(src, dest, JSVAL_TYPE_BIGINT, fail);
}
//}}} check_macroassembler_style
// ===============================================================
void MacroAssembler::ensureDouble(const ValueOperand& source,
FloatRegister dest, Label* failure) {
Label isDouble, done;
{
ScratchTagScope tag(*this, source);
splitTagForTest(source, tag);
branchTestDouble(Assembler::Equal, tag, &isDouble);
branchTestInt32(Assembler::NotEqual, tag, failure);
}
convertInt32ToDouble(source.payloadOrValueReg(), dest);
jump(&done);
bind(&isDouble);
unboxDouble(source, dest);
bind(&done);
}
#ifndef JS_CODEGEN_ARM64
template <typename T>
void MacroAssembler::branchTestStackPtr(Condition cond, T t, Label* label) {
branchTestPtr(cond, getStackPointer(), t, label);
}
template <typename T>
void MacroAssembler::branchStackPtr(Condition cond, T rhs, Label* label) {
branchPtr(cond, getStackPointer(), rhs, label);
}
template <typename T>
void MacroAssembler::branchStackPtrRhs(Condition cond, T lhs, Label* label) {
branchPtr(cond, lhs, getStackPointer(), label);
}
template <typename T>
void MacroAssembler::addToStackPtr(T t) {
addPtr(t, getStackPointer());
}
template <typename T>
void MacroAssembler::addStackPtrTo(T t) {
addPtr(getStackPointer(), t);
}
void MacroAssembler::reserveStack(uint32_t amount) {
subFromStackPtr(Imm32(amount));
adjustFrame(amount);
}
#endif // !JS_CODEGEN_ARM64
void MacroAssembler::loadObjClassUnsafe(Register obj, Register dest) {
loadPtr(Address(obj, JSObject::offsetOfShape()), dest);
loadPtr(Address(dest, Shape::offsetOfBaseShape()), dest);
loadPtr(Address(dest, BaseShape::offsetOfClasp()), dest);
}
void MacroAssembler::loadObjShapeUnsafe(Register obj, Register dest) {
loadPtr(Address(obj, JSObject::offsetOfShape()), dest);
}
template <typename EmitPreBarrier>
void MacroAssembler::storeObjShape(Register shape, Register obj,
EmitPreBarrier emitPreBarrier) {
MOZ_ASSERT(shape != obj);
Address shapeAddr(obj, JSObject::offsetOfShape());
emitPreBarrier(*this, shapeAddr);
storePtr(shape, shapeAddr);
}
template <typename EmitPreBarrier>
void MacroAssembler::storeObjShape(Shape* shape, Register obj,
EmitPreBarrier emitPreBarrier) {
Address shapeAddr(obj, JSObject::offsetOfShape());
emitPreBarrier(*this, shapeAddr);
storePtr(ImmGCPtr(shape), shapeAddr);
}
void MacroAssembler::loadObjProto(Register obj, Register dest) {
loadPtr(Address(obj, JSObject::offsetOfShape()), dest);
loadPtr(Address(dest, Shape::offsetOfBaseShape()), dest);
loadPtr(Address(dest, BaseShape::offsetOfProto()), dest);
}
void MacroAssembler::loadStringLength(Register str, Register dest) {
load32(Address(str, JSString::offsetOfLength()), dest);
}
void MacroAssembler::assertStackAlignment(uint32_t alignment,
int32_t offset /* = 0 */) {
#ifdef DEBUG
Label ok, bad;
MOZ_ASSERT(mozilla::IsPowerOfTwo(alignment));
// Wrap around the offset to be a non-negative number.
offset %= alignment;
if (offset < 0) {
offset += alignment;
}
// Test if each bit from offset is set.
uint32_t off = offset;
while (off) {
uint32_t lowestBit = 1 << mozilla::CountTrailingZeroes32(off);
branchTestStackPtr(Assembler::Zero, Imm32(lowestBit), &bad);
off ^= lowestBit;
}
// Check that all remaining bits are zero.
branchTestStackPtr(Assembler::Zero, Imm32((alignment - 1) ^ offset), &ok);
bind(&bad);
breakpoint();
bind(&ok);
#endif
}
void MacroAssembler::storeCallBoolResult(Register reg) {
convertBoolToInt32(ReturnReg, reg);
}
void MacroAssembler::storeCallInt32Result(Register reg) {
#if JS_BITS_PER_WORD == 32
storeCallPointerResult(reg);
#else
// Ensure the upper 32 bits are cleared.
move32(ReturnReg, reg);
#endif
}
void MacroAssembler::storeCallResultValue(AnyRegister dest, JSValueType type) {
unboxValue(JSReturnOperand, dest, type);
}
void MacroAssembler::storeCallResultValue(TypedOrValueRegister dest) {
if (dest.hasValue()) {
storeCallResultValue(dest.valueReg());
} else {
storeCallResultValue(dest.typedReg(), ValueTypeFromMIRType(dest.type()));
}
}
} // namespace jit
} // namespace js
#endif /* jit_MacroAssembler_inl_h */