CodeGenerator.h - mozsearch

mozilla-central/js/src/jit/CodeGenerator.h (file symbol)

Enable keyboard shortcuts

Source code

Revision control

Copy as Markdown

Other Tools

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-

 * vim: set ts=8 sts=2 et sw=2 tw=80:

 * This Source Code Form is subject to the terms of the Mozilla Public

 * License, v. 2.0. If a copy of the MPL was not distributed with this

 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifndef jit_CodeGenerator_h

#define jit_CodeGenerator_h

#include "jit/PerfSpewer.h"

#include "js/Prefs.h"

#include "js/ScalarType.h"  // js::Scalar::Type

#if defined(JS_CODEGEN_X86)

#  include "jit/x86/CodeGenerator-x86.h"

#elif defined(JS_CODEGEN_X64)

#  include "jit/x64/CodeGenerator-x64.h"

#elif defined(JS_CODEGEN_ARM)

#  include "jit/arm/CodeGenerator-arm.h"

#elif defined(JS_CODEGEN_ARM64)

#  include "jit/arm64/CodeGenerator-arm64.h"

#elif defined(JS_CODEGEN_MIPS32)

#  include "jit/mips32/CodeGenerator-mips32.h"

#elif defined(JS_CODEGEN_MIPS64)

#  include "jit/mips64/CodeGenerator-mips64.h"

#elif defined(JS_CODEGEN_LOONG64)

#  include "jit/loong64/CodeGenerator-loong64.h"

#elif defined(JS_CODEGEN_RISCV64)

#  include "jit/riscv64/CodeGenerator-riscv64.h"

#elif defined(JS_CODEGEN_WASM32)

#  include "jit/wasm32/CodeGenerator-wasm32.h"

#elif defined(JS_CODEGEN_NONE)

#  include "jit/none/CodeGenerator-none.h"

#else

#  error "Unknown architecture!"

#endif

namespace js {

namespace wasm {

class Decoder;

class StackMaps;

}  // namespace wasm

namespace jit {

class WarpSnapshot;

template <typename Fn, Fn fn, class ArgSeq, class StoreOutputTo>

class OutOfLineCallVM;

enum class SwitchTableType { Inline, OutOfLine };

template <SwitchTableType tableType>

class OutOfLineSwitch;

class OutOfLineTestObject;

class OutOfLineNewArray;

class OutOfLineNewObject;

class CheckOverRecursedFailure;

class OutOfLineUnboxFloatingPoint;

class OutOfLineStoreElementHole;

class OutOfLineTypeOfV;

class OutOfLineTypeOfIsNonPrimitiveV;

class OutOfLineTypeOfIsNonPrimitiveO;

class OutOfLineUpdateCache;

class OutOfLineICFallback;

class OutOfLineCallPostWriteBarrier;

class OutOfLineCallPostWriteElementBarrier;

class OutOfLineElementPostWriteBarrier;

class OutOfLineIsCallable;

class OutOfLineIsConstructor;

class OutOfLineRegExpMatcher;

class OutOfLineRegExpSearcher;

class OutOfLineRegExpExecMatch;

class OutOfLineRegExpExecTest;

class OutOfLineRegExpPrototypeOptimizable;

class OutOfLineRegExpInstanceOptimizable;

class OutOfLineNaNToZero;

class OutOfLineResumableWasmTrap;

class OutOfLineAbortingWasmTrap;

class OutOfLineGuardNumberToIntPtrIndex;

class OutOfLineBoxNonStrictThis;

class OutOfLineArrayPush;

class OutOfLineAtomizeSlot;

class OutOfLineWasmCallPostWriteBarrierImmediate;

class OutOfLineWasmCallPostWriteBarrierIndex;

class OutOfLineWasmNewStruct;

class OutOfLineWasmNewArray;

class CodeGenerator final : public CodeGeneratorSpecific {

  [[nodiscard]] bool generateBody();

  ConstantOrRegister toConstantOrRegister(LInstruction* lir, size_t n,

                                          MIRType type);

#ifdef CHECK_OSIPOINT_REGISTERS

  void resetOsiPointRegs(LSafepoint* safepoint);

  bool shouldVerifyOsiPointRegs(LSafepoint* safepoint);

  void verifyOsiPointRegs(LSafepoint* safepoint);

#endif

  void callVMInternal(VMFunctionId id, LInstruction* ins);

  template <typename Fn, Fn fn>

  void callVM(LInstruction* ins);

  template <typename Fn, Fn fn, class ArgSeq, class StoreOutputTo>

  inline OutOfLineCode* oolCallVM(LInstruction* ins, const ArgSeq& args,

                                  const StoreOutputTo& out);

  template <typename LCallIns>

  void emitCallNative(LCallIns* call, JSNative native, Register argContextReg,

                      Register argUintNReg, Register argVpReg, Register tempReg,

                      uint32_t unusedStack);

  template <typename LCallIns>

  void emitCallNative(LCallIns* call, JSNative native);

 public:

  CodeGenerator(MIRGenerator* gen, LIRGraph* graph,

                MacroAssembler* masm = nullptr);

  ~CodeGenerator();

  [[nodiscard]] bool generate();

  [[nodiscard]] bool generateWasm(

      wasm::CallIndirectId callIndirectId, wasm::BytecodeOffset trapOffset,

      const wasm::ArgTypeVector& argTys, const RegisterOffsets& trapExitLayout,

      size_t trapExitLayoutNumWords, wasm::FuncOffsets* offsets,

      wasm::StackMaps* stackMaps, wasm::Decoder* decoder);

  [[nodiscard]] bool link(JSContext* cx, const WarpSnapshot* snapshot);

  void emitOOLTestObject(Register objreg, Label* ifTruthy, Label* ifFalsy,

                         Register scratch);

  void emitTypeOfCheck(JSValueType type, Register tag, Register output,

                       Label* done, Label* oolObject);

  void emitTypeOfJSType(JSValueType type, Register output);

  void emitTypeOfObject(Register obj, Register output, Label* done);

  void emitTypeOfIsObject(MTypeOfIs* mir, Register obj, Register output,

                          Label* success, Label* fail, Label* slowCheck);

  void emitTypeOfIsObjectOOL(MTypeOfIs* mir, Register obj, Register output);

  template <typename Fn, Fn fn, class ArgSeq, class StoreOutputTo>

  void visitOutOfLineCallVM(

      OutOfLineCallVM<Fn, fn, ArgSeq, StoreOutputTo>* ool);

  void visitOutOfLineRegExpMatcher(OutOfLineRegExpMatcher* ool);

  void visitOutOfLineRegExpSearcher(OutOfLineRegExpSearcher* ool);

  void visitOutOfLineRegExpExecMatch(OutOfLineRegExpExecMatch* ool);

  void visitOutOfLineRegExpExecTest(OutOfLineRegExpExecTest* ool);

  void visitOutOfLineRegExpPrototypeOptimizable(

      OutOfLineRegExpPrototypeOptimizable* ool);

  void visitOutOfLineRegExpInstanceOptimizable(

      OutOfLineRegExpInstanceOptimizable* ool);

  void visitOutOfLineTypeOfV(OutOfLineTypeOfV* ool);

  void visitOutOfLineTypeOfIsNonPrimitiveV(OutOfLineTypeOfIsNonPrimitiveV* ool);

  void visitOutOfLineTypeOfIsNonPrimitiveO(OutOfLineTypeOfIsNonPrimitiveO* ool);

  template <SwitchTableType tableType>

  void visitOutOfLineSwitch(OutOfLineSwitch<tableType>* ool);

  void visitOutOfLineIsCallable(OutOfLineIsCallable* ool);

  void visitOutOfLineIsConstructor(OutOfLineIsConstructor* ool);

  void visitOutOfLineNaNToZero(OutOfLineNaNToZero* ool);

  void visitOutOfLineResumableWasmTrap(OutOfLineResumableWasmTrap* ool);

  void visitOutOfLineAbortingWasmTrap(OutOfLineAbortingWasmTrap* ool);

  void visitCheckOverRecursedFailure(CheckOverRecursedFailure* ool);

  void visitOutOfLineUnboxFloatingPoint(OutOfLineUnboxFloatingPoint* ool);

  void visitOutOfLineStoreElementHole(OutOfLineStoreElementHole* ool);

  void visitOutOfLineBoxNonStrictThis(OutOfLineBoxNonStrictThis* ool);

  void visitOutOfLineICFallback(OutOfLineICFallback* ool);

  void visitOutOfLineCallPostWriteBarrier(OutOfLineCallPostWriteBarrier* ool);

  void visitOutOfLineCallPostWriteElementBarrier(

      OutOfLineCallPostWriteElementBarrier* ool);

  void visitOutOfLineElementPostWriteBarrier(

      OutOfLineElementPostWriteBarrier* ool);

  void visitOutOfLineNewArray(OutOfLineNewArray* ool);

  void visitOutOfLineNewObject(OutOfLineNewObject* ool);

  void visitOutOfLineGuardNumberToIntPtrIndex(

      OutOfLineGuardNumberToIntPtrIndex* ool);

  void visitOutOfLineArrayPush(OutOfLineArrayPush* ool);

  void visitOutOfLineAtomizeSlot(OutOfLineAtomizeSlot* ool);

  void visitOutOfLineWasmCallPostWriteBarrierImmediate(

      OutOfLineWasmCallPostWriteBarrierImmediate* ool);

  void visitOutOfLineWasmCallPostWriteBarrierIndex(

      OutOfLineWasmCallPostWriteBarrierIndex* ool);

  void callWasmStructAllocFun(LInstruction* lir, wasm::SymbolicAddress fun,

                              Register typeDefData, Register output);

  void visitOutOfLineWasmNewStruct(OutOfLineWasmNewStruct* ool);

  void callWasmArrayAllocFun(LInstruction* lir, wasm::SymbolicAddress fun,

                             Register numElements, Register typeDefData,

                             Register output,

                             wasm::BytecodeOffset bytecodeOffset);

  void visitOutOfLineWasmNewArray(OutOfLineWasmNewArray* ool);

 private:

  void emitPostWriteBarrier(const LAllocation* obj);

  void emitPostWriteBarrier(Register objreg);

  void emitPostWriteBarrierS(Address address, Register prev, Register next);

  void emitElementPostWriteBarrier(MInstruction* mir,

                                   const LiveRegisterSet& liveVolatileRegs,

                                   Register obj, const LAllocation* index,

                                   Register scratch,

                                   const ConstantOrRegister& val,

                                   int32_t indexDiff = 0);

  template <class LPostBarrierType, MIRType nurseryType>

  void visitPostWriteBarrierCommon(LPostBarrierType* lir, OutOfLineCode* ool);

  template <class LPostBarrierType>

  void visitPostWriteBarrierCommonV(LPostBarrierType* lir, OutOfLineCode* ool);

  void emitCallInvokeFunction(LInstruction* call, Register callereg,

                              bool isConstructing, bool ignoresReturnValue,

                              uint32_t argc, uint32_t unusedStack);

  template <typename T>

  void emitApplyGeneric(T* apply);

  template <typename T>

  void emitCallInvokeFunction(T* apply);

  void emitAllocateSpaceForApply(Register argcreg, Register scratch);

  void emitAllocateSpaceForConstructAndPushNewTarget(

      Register argcreg, Register newTargetAndScratch);

  void emitCopyValuesForApply(Register argvSrcBase, Register argvIndex,

                              Register copyreg, size_t argvSrcOffset,

                              size_t argvDstOffset);

  void emitRestoreStackPointerFromFP();

  void emitPushArguments(Register argcreg, Register scratch, Register copyreg,

                         uint32_t extraFormals);

  void emitPushArrayAsArguments(Register tmpArgc, Register srcBaseAndArgc,

                                Register scratch, size_t argvSrcOffset);

  void emitPushArguments(LApplyArgsGeneric* apply);

  void emitPushArguments(LApplyArgsObj* apply);

  void emitPushArguments(LApplyArrayGeneric* apply);

  void emitPushArguments(LConstructArgsGeneric* construct);

  void emitPushArguments(LConstructArrayGeneric* construct);

  template <typename T>

  void emitApplyNative(T* apply);

  template <typename T>

  void emitAlignStackForApplyNative(T* apply, Register argc);

  template <typename T>

  void emitPushNativeArguments(T* apply);

  template <typename T>

  void emitPushArrayAsNativeArguments(T* apply);

  void emitPushArguments(LApplyArgsNative* apply);

  void emitPushArguments(LApplyArgsObjNative* apply);

  void emitPushArguments(LApplyArrayNative* apply);

  void emitPushArguments(LConstructArgsNative* construct);

  void emitPushArguments(LConstructArrayNative* construct);

  template <typename T>

  void emitApplyArgsGuard(T* apply);

  template <typename T>

  void emitApplyArgsObjGuard(T* apply);

  template <typename T>

  void emitApplyArrayGuard(T* apply);

  template <class GetInlinedArgument>

  void emitGetInlinedArgument(GetInlinedArgument* lir, Register index,

                              ValueOperand output);

  void emitMaybeAtomizeSlot(LInstruction* ins, Register stringReg,

                            Address slotAddr, TypedOrValueRegister dest);

  using RegisterOrInt32 = mozilla::Variant<Register, int32_t>;

  static RegisterOrInt32 ToRegisterOrInt32(const LAllocation* allocation);

#ifdef DEBUG

  void emitAssertArgumentsSliceBounds(const RegisterOrInt32& begin,

                                      const RegisterOrInt32& count,

                                      Register numActualArgs);

#endif

  template <class ArgumentsSlice>

  void emitNewArray(ArgumentsSlice* lir, const RegisterOrInt32& count,

                    Register output, Register temp);

  void visitNewArrayCallVM(LNewArray* lir);

  void visitNewObjectVMCall(LNewObject* lir);

  void emitConcat(LInstruction* lir, Register lhs, Register rhs,

                  Register output);

  void emitInstanceOf(LInstruction* ins, Register protoReg);

  void loadJSScriptForBlock(MBasicBlock* block, Register reg);

  void loadOutermostJSScript(Register reg);

#ifdef DEBUG

  void emitAssertResultV(const ValueOperand output, const MDefinition* mir);

  void emitAssertGCThingResult(Register input, const MDefinition* mir);

#endif

#ifdef DEBUG

  void emitDebugForceBailing(LInstruction* lir);

#endif

  IonScriptCounts* extractScriptCounts() {

    IonScriptCounts* counts = scriptCounts_;

    scriptCounts_ = nullptr;  // prevent delete in dtor

    return counts;

  void addGetPropertyCache(LInstruction* ins, LiveRegisterSet liveRegs,

                           TypedOrValueRegister value,

                           const ConstantOrRegister& id, ValueOperand output);

  void addSetPropertyCache(LInstruction* ins, LiveRegisterSet liveRegs,

                           Register objReg, Register temp,

                           const ConstantOrRegister& id,

                           const ConstantOrRegister& value, bool strict);

  template <class IteratorObject, class OrderedHashTable>

  void emitGetNextEntryForIterator(LGetNextEntryForIterator* lir);

  template <class OrderedHashTable>

  void emitLoadIteratorValues(Register result, Register temp, Register front);

  void emitStringToInt64(LInstruction* lir, Register input, Register64 output);

  OutOfLineCode* createBigIntOutOfLine(LInstruction* lir, Scalar::Type type,

                                       Register64 input, Register output);

  void emitCreateBigInt(LInstruction* lir, Scalar::Type type, Register64 input,

                        Register output, Register maybeTemp);

  template <size_t NumDefs>

  void emitIonToWasmCallBase(LIonToWasmCallBase<NumDefs>* lir);

  IonScriptCounts* maybeCreateScriptCounts();

  void emitWasmCompareAndSelect(LWasmCompareAndSelect* ins);

  template <typename InstructionWithMaybeTrapSite, class AddressOrBaseIndex>

  void emitWasmValueLoad(InstructionWithMaybeTrapSite* ins, MIRType type,

                         MWideningOp wideningOp, AddressOrBaseIndex addr,

                         AnyRegister dst);

  template <typename InstructionWithMaybeTrapSite, class AddressOrBaseIndex>

  void emitWasmValueStore(InstructionWithMaybeTrapSite* ins, MIRType type,

                          MNarrowingOp narrowingOp, AnyRegister src,

                          AddressOrBaseIndex addr);

  void testValueTruthyForType(JSValueType type, ScratchTagScope& tag,

                              const ValueOperand& value, Register tempToUnbox,

                              Register temp, FloatRegister floatTemp,

                              Label* ifTruthy, Label* ifFalsy,

                              OutOfLineTestObject* ool, bool skipTypeTest);

  // Test whether value is truthy or not and jump to the corresponding label.

  // The control flow falls through when the object is truthy, as an

  // optimization.

  void testValueTruthy(const ValueOperand& value, Register tempToUnbox,

                       Register temp, FloatRegister floatTemp,

                       const TypeDataList& observedTypes, Label* ifTruthy,

                       Label* ifFalsy, OutOfLineTestObject* ool);

  // This function behaves like testObjectEmulatesUndefined with the exception

  // that it can choose to let control flow fall through when the object

  // doesn't emulate undefined, as an optimization. Use the regular

  // testObjectEmulatesUndefined when it's required to branch to one of the

  // two labels.

  void testObjectEmulatesUndefinedKernel(Register objreg,

                                         Label* ifEmulatesUndefined,

                                         Label* ifDoesntEmulateUndefined,

                                         Register scratch,

                                         OutOfLineTestObject* ool);

  // Test whether an object emulates |undefined|.  If it does, jump to

  // |ifEmulatesUndefined|; the caller is responsible for binding this label.

  // If it doesn't, fall through; the label |ifDoesntEmulateUndefined| (which

  // must be initially unbound) will be bound at this point.

  void branchTestObjectEmulatesUndefined(Register objreg,

                                         Label* ifEmulatesUndefined,

                                         Label* ifDoesntEmulateUndefined,

                                         Register scratch,

                                         OutOfLineTestObject* ool);

  // Test whether an object emulates |undefined|, and jump to the

  // corresponding label.

//

  // This method should be used when subsequent code can't be laid out in a

  // straight line; if it can, branchTest* should be used instead.

  void testObjectEmulatesUndefined(Register objreg, Label* ifEmulatesUndefined,

                                   Label* ifDoesntEmulateUndefined,

                                   Register scratch, OutOfLineTestObject* ool);

  void emitStoreElementTyped(const LAllocation* value, MIRType valueType,

                             Register elements, const LAllocation* index);

  // Bailout if an element about to be written to is a hole.

  void emitStoreHoleCheck(Register elements, const LAllocation* index,

                          LSnapshot* snapshot);

  void emitAssertRangeI(MIRType type, const Range* r, Register input);

  void emitAssertRangeD(const Range* r, FloatRegister input,

                        FloatRegister temp);

  void maybeEmitGlobalBarrierCheck(const LAllocation* maybeGlobal,

                                   OutOfLineCode* ool);

  void incrementWarmUpCounter(AbsoluteAddress warmUpCount, JSScript* script,

                              Register tmp);

  Vector<CodeOffset, 0, JitAllocPolicy> ionScriptLabels_;

  // Used to bake in a pointer into the IonScript's list of nursery objects, for

  // MNurseryObject codegen.

  struct NurseryObjectLabel {

    CodeOffset offset;

    uint32_t nurseryIndex;

    NurseryObjectLabel(CodeOffset offset, uint32_t nurseryIndex)

        : offset(offset), nurseryIndex(nurseryIndex) {}

};

  Vector<NurseryObjectLabel, 0, JitAllocPolicy> ionNurseryObjectLabels_;

  void branchIfInvalidated(Register temp, Label* invalidated);

#ifdef DEBUG

  void emitDebugResultChecks(LInstruction* ins);

  void emitGCThingResultChecks(LInstruction* lir, MDefinition* mir);

  void emitValueResultChecks(LInstruction* lir, MDefinition* mir);

#endif

  // Script counts created during code generation.

  IonScriptCounts* scriptCounts_;

  IonPerfSpewer perfSpewer_;

  // Bit mask of JitZone stubs that are to be read-barriered.

  uint32_t zoneStubsToReadBarrier_;

#ifdef FUZZING_JS_FUZZILLI

  void emitFuzzilliHashDouble(FloatRegister floatDouble, Register scratch,

                              Register output);

  void emitFuzzilliHashObject(LInstruction* lir, Register obj, Register output);

  void emitFuzzilliHashBigInt(Register bigInt, Register output);

#endif

#define LIR_OP(op) void visit##op(L##op* ins);

  LIR_OPCODE_LIST(LIR_OP)

#undef LIR_OP

  // In debug mode, we need to validate that we've not made a mistake with the

  // fuse.

  void assertObjectDoesNotEmulateUndefined(Register input, Register temp,

                                           const MInstruction* mir);

  // Enumerates the fuses that a code generation can depend on. These will

  // be mapped to an actual fuse by validateAndRegisterFuseDependencies.

  enum class FuseDependencyKind {

    HasSeenObjectEmulateUndefinedFuse,

    OptimizeGetIteratorFuse,

};

  // The set of fuses this code generation depends on.

  mozilla::EnumSet<FuseDependencyKind> fuseDependencies;

  // Register a dependency on the HasSeenObjectEmulateUndefined fuse.

  void addHasSeenObjectEmulateUndefinedFuseDependency() {

    fuseDependencies += FuseDependencyKind::HasSeenObjectEmulateUndefinedFuse;

  void addOptimizeGetIteratorFuseDependency() {

    fuseDependencies += FuseDependencyKind::OptimizeGetIteratorFuse;

  // Called during linking on main-thread: Ensures that the fuses are still

  // intact, and registers a script dependency on a specific fuse before

  // finishing compilation.

  void validateAndRegisterFuseDependencies(JSContext* cx, HandleScript script,

                                           bool* isValid);

  // Return true if the fuse is intact, andd if the fuse is intact note the

  // dependency

  bool hasSeenObjectEmulateUndefinedFuseIntactAndDependencyNoted() {

    if (!JS::Prefs::use_emulates_undefined_fuse()) {

      // if we're not active, simply pretend the fuse is popped.

      return false;

    bool intact = gen->outerInfo().hasSeenObjectEmulateUndefinedFuseIntact();

    if (intact) {

      addHasSeenObjectEmulateUndefinedFuseDependency();

    return intact;

};

class OutOfLineResumableWasmTrap : public OutOfLineCodeBase<CodeGenerator> {

  LInstruction* lir_;

  size_t framePushed_;

  wasm::BytecodeOffset bytecodeOffset_;

  wasm::Trap trap_;

 public:

  OutOfLineResumableWasmTrap(LInstruction* lir, size_t framePushed,

                             wasm::BytecodeOffset bytecodeOffset,

                             wasm::Trap trap)

      : lir_(lir),

        framePushed_(framePushed),

        bytecodeOffset_(bytecodeOffset),

        trap_(trap) {}

  void accept(CodeGenerator* codegen) override {

    codegen->visitOutOfLineResumableWasmTrap(this);

  LInstruction* lir() const { return lir_; }

  size_t framePushed() const { return framePushed_; }

  wasm::BytecodeOffset bytecodeOffset() const { return bytecodeOffset_; }

  wasm::Trap trap() const { return trap_; }

};

class OutOfLineAbortingWasmTrap : public OutOfLineCodeBase<CodeGenerator> {

  wasm::BytecodeOffset bytecodeOffset_;

  wasm::Trap trap_;

 public:

  OutOfLineAbortingWasmTrap(wasm::BytecodeOffset bytecodeOffset,

                            wasm::Trap trap)

      : bytecodeOffset_(bytecodeOffset), trap_(trap) {}

  void accept(CodeGenerator* codegen) override {

    codegen->visitOutOfLineAbortingWasmTrap(this);

  wasm::BytecodeOffset bytecodeOffset() const { return bytecodeOffset_; }

  wasm::Trap trap() const { return trap_; }

};

}  // namespace jit

}  // namespace js

#endif /* jit_CodeGenerator_h */