Source code
Revision control
Copy as Markdown
Other Tools
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: set ts=8 sts=2 et sw=2 tw=80:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
#ifndef jit_JitScript_h
#define jit_JitScript_h
#include "mozilla/Assertions.h"
#include "mozilla/Atomics.h"
#include "mozilla/Attributes.h"
#include "mozilla/HashFunctions.h"
#include "mozilla/LinkedList.h"
#include "mozilla/Maybe.h"
#include "mozilla/MemoryReporting.h"
#include <stddef.h>
#include <stdint.h>
#include "jstypes.h"
#include "NamespaceImports.h"
#include "ds/LifoAlloc.h"
#include "gc/Barrier.h"
#include "jit/BaselineIC.h"
#include "js/TypeDecls.h"
#include "js/UniquePtr.h"
#include "js/Vector.h"
#include "util/TrailingArray.h"
#include "vm/EnvironmentObject.h"
class JS_PUBLIC_API JSScript;
class JS_PUBLIC_API JSTracer;
struct JS_PUBLIC_API JSContext;
namespace JS {
class Zone;
}
namespace js {
class SystemAllocPolicy;
namespace gc {
class AllocSite;
}
namespace jit {
class BaselineScript;
class ICStubSpace;
class InliningRoot;
class IonScript;
class JitScript;
class JitZone;
// Magic BaselineScript value indicating Baseline compilation has been disabled.
static constexpr uintptr_t BaselineDisabledScript = 0x1;
static BaselineScript* const BaselineDisabledScriptPtr =
reinterpret_cast<BaselineScript*>(BaselineDisabledScript);
// Magic IonScript values indicating Ion compilation has been disabled or the
// script is being Ion-compiled off-thread.
static constexpr uintptr_t IonDisabledScript = 0x1;
static constexpr uintptr_t IonCompilingScript = 0x2;
static IonScript* const IonDisabledScriptPtr =
reinterpret_cast<IonScript*>(IonDisabledScript);
static IonScript* const IonCompilingScriptPtr =
reinterpret_cast<IonScript*>(IonCompilingScript);
/* [SMDOC] ICScript Lifetimes
*
* An ICScript owns an array of ICEntries, each of which owns a linked
* list of ICStubs.
*
* A JitScript contains an embedded ICScript. If it has done any trial
* inlining, it also owns an InliningRoot. The InliningRoot owns all
* of the ICScripts that have been created for inlining into the
* corresponding JitScript. This ties the lifetime of the inlined
* ICScripts to the lifetime of the JitScript itself.
*
* We store pointers to ICScripts in two other places: on the stack in
* BaselineFrame, and in IC stubs for CallInlinedFunction.
*
* The ICScript pointer in a BaselineFrame either points to the
* ICScript embedded in the JitScript for that frame, or to an inlined
* ICScript owned by a caller. In each case, there must be a frame on
* the stack corresponding to the JitScript that owns the current
* ICScript, which will keep the ICScript alive.
*
* Each ICStub is owned by an ICScript and, indirectly, a
* JitScript. An ICStub that uses CallInlinedFunction contains an
* ICScript for use by the callee. The ICStub and the callee ICScript
* are always owned by the same JitScript, so the callee ICScript will
* not be freed while the ICStub is alive.
*
* The lifetime of an ICScript is independent of the lifetimes of the
* BaselineScript and IonScript/WarpScript to which it
* corresponds. They can be destroyed and recreated, and the ICScript
* will remain valid.
*
* When we discard JIT code, we mark ICScripts that are active on the stack as
* active and then purge all of the inactive ICScripts. We also purge ICStubs,
* including the CallInlinedFunction stub at the trial inining call site, and
* reset the ICStates to allow trial inlining again later.
*
* If there's a BaselineFrame for an inlined ICScript, we'll preserve both this
* ICScript and the IC chain for the call site in the caller's ICScript.
* See ICScript::purgeStubs and ICScript::purgeInactiveICScripts.
*/
class alignas(uintptr_t) ICScript final : public TrailingArray<ICScript> {
public:
ICScript(uint32_t warmUpCount, Offset fallbackStubsOffset, Offset endOffset,
uint32_t depth, uint32_t bytecodeSize,
InliningRoot* inliningRoot = nullptr)
: inliningRoot_(inliningRoot),
warmUpCount_(warmUpCount),
fallbackStubsOffset_(fallbackStubsOffset),
endOffset_(endOffset),
depth_(depth),
bytecodeSize_(bytecodeSize) {}
~ICScript();
bool isInlined() const { return depth_ > 0; }
void initICEntries(JSContext* cx, JSScript* script);
ICEntry& icEntry(size_t index) {
MOZ_ASSERT(index < numICEntries());
return icEntries()[index];
}
ICFallbackStub* fallbackStub(size_t index) {
MOZ_ASSERT(index < numICEntries());
return fallbackStubs() + index;
}
ICEntry* icEntryForStub(const ICFallbackStub* stub) {
size_t index = stub - fallbackStubs();
MOZ_ASSERT(index < numICEntries());
return &icEntry(index);
}
ICFallbackStub* fallbackStubForICEntry(const ICEntry* entry) {
size_t index = entry - icEntries();
MOZ_ASSERT(index < numICEntries());
return fallbackStub(index);
}
InliningRoot* inliningRoot() const { return inliningRoot_; }
uint32_t depth() const { return depth_; }
uint32_t bytecodeSize() const { return bytecodeSize_; }
void resetWarmUpCount(uint32_t count) { warmUpCount_ = count; }
static constexpr size_t offsetOfFirstStub(uint32_t entryIndex) {
return sizeof(ICScript) + entryIndex * sizeof(ICEntry) +
ICEntry::offsetOfFirstStub();
}
static constexpr Offset offsetOfWarmUpCount() {
return offsetof(ICScript, warmUpCount_);
}
static constexpr Offset offsetOfDepth() { return offsetof(ICScript, depth_); }
static constexpr Offset offsetOfICEntries() { return sizeof(ICScript); }
uint32_t numICEntries() const {
return numElements<ICEntry>(icEntriesOffset(), fallbackStubsOffset());
}
ICEntry* interpreterICEntryFromPCOffset(uint32_t pcOffset);
ICEntry& icEntryFromPCOffset(uint32_t pcOffset);
[[nodiscard]] bool addInlinedChild(JSContext* cx,
js::UniquePtr<ICScript> child,
uint32_t pcOffset);
ICScript* findInlinedChild(uint32_t pcOffset);
void removeInlinedChild(uint32_t pcOffset);
bool hasInlinedChild(uint32_t pcOffset);
void purgeStubs(Zone* zone, ICStubSpace& newStubSpace);
void purgeInactiveICScripts();
bool active() const { return active_; }
void setActive() { active_ = true; }
void resetActive() { active_ = false; }
gc::AllocSite* getOrCreateAllocSite(JSScript* outerScript, uint32_t pcOffset);
void prepareForDestruction(Zone* zone);
void trace(JSTracer* trc);
bool traceWeak(JSTracer* trc);
#ifdef DEBUG
mozilla::HashNumber hash();
#endif
private:
class CallSite {
public:
CallSite(ICScript* callee, uint32_t pcOffset)
: callee_(callee), pcOffset_(pcOffset) {}
ICScript* callee_;
uint32_t pcOffset_;
};
// If this ICScript was created for trial inlining or has another
// ICScript inlined into it, a pointer to the root of the inlining
// tree. Otherwise, nullptr.
InliningRoot* inliningRoot_ = nullptr;
// ICScripts that have been inlined into this ICScript.
js::UniquePtr<Vector<CallSite>> inlinedChildren_;
// List of allocation sites referred to by ICs in this script.
static constexpr size_t AllocSiteChunkSize = 256;
LifoAlloc allocSitesSpace_{AllocSiteChunkSize};
Vector<gc::AllocSite*, 0, SystemAllocPolicy> allocSites_;
// Number of times this copy of the script has been called or has had
// backedges taken. Reset if the script's JIT code is forcibly discarded.
// See also the ScriptWarmUpData class.
mozilla::Atomic<uint32_t, mozilla::Relaxed> warmUpCount_ = {};
// The offset of the ICFallbackStub array.
Offset fallbackStubsOffset_;
// The size of this allocation.
Offset endOffset_;
// The inlining depth of this ICScript. 0 for the inlining root.
uint32_t depth_;
// Bytecode size of the JSScript corresponding to this ICScript.
uint32_t bytecodeSize_;
// Flag set when discarding JIT code to indicate this script is on the stack
// and should not be discarded.
bool active_ = false;
Offset icEntriesOffset() const { return offsetOfICEntries(); }
Offset fallbackStubsOffset() const { return fallbackStubsOffset_; }
Offset endOffset() const { return endOffset_; }
public:
ICEntry* icEntries() { return offsetToPointer<ICEntry>(icEntriesOffset()); }
private:
ICFallbackStub* fallbackStubs() {
return offsetToPointer<ICFallbackStub>(fallbackStubsOffset());
}
JitScript* outerJitScript();
friend class JitScript;
};
// [SMDOC] JitScript
//
// JitScript stores type inference data, Baseline ICs and other JIT-related data
// for a script. Scripts with a JitScript can run in the Baseline Interpreter.
//
// IC Data
// =======
// All IC data for Baseline (Interpreter and JIT) is stored in an ICScript. Each
// JitScript contains an ICScript as the last field. Additional free-standing
// ICScripts may be created during trial inlining. Ion has its own IC chains
// stored in IonScript.
//
// For each IC we store an ICEntry, which points to the first ICStub in the
// chain, and an ICFallbackStub. Note that multiple stubs in the same zone can
// share Baseline IC code. This works because the stub data is stored in the
// ICStub instead of baked in in the stub code.
//
// Storing this separate from BaselineScript allows us to use the same ICs in
// the Baseline Interpreter and Baseline JIT. It also simplifies debug mode OSR
// because the JitScript can be reused when we have to recompile the
// BaselineScript.
//
// An ICScript contains a list of IC entries and a list of fallback stubs.
// There's one ICEntry and ICFallbackStub for each JOF_IC bytecode op.
//
// The ICScript also contains the warmUpCount for the script.
//
// Inlining Data
// =============
// JitScript also contains a list of Warp compilations inlining this script, for
// invalidation.
//
// Memory Layout
// =============
// JitScript contains an ICScript as the last field. ICScript has trailing
// (variable length) arrays for ICEntry and ICFallbackStub. The memory layout is
// as follows:
//
// Item | Offset
// ------------------------+------------------------
// JitScript | 0
// -->ICScript (field) |
// ICEntry[] | icEntriesOffset()
// ICFallbackStub[] | fallbackStubsOffset()
//
// These offsets are also used to compute numICEntries.
class alignas(uintptr_t) JitScript final
: public mozilla::LinkedListElement<JitScript>,
public TrailingArray<JitScript> {
friend class ::JSScript;
// Profile string used by the profiler for Baseline Interpreter frames.
const char* profileString_ = nullptr;
HeapPtr<JSScript*> owningScript_;
// Baseline code for the script. Either nullptr, BaselineDisabledScriptPtr or
// a valid BaselineScript*.
GCStructPtr<BaselineScript*> baselineScript_;
// Ion code for this script. Either nullptr, IonDisabledScriptPtr,
// IonCompilingScriptPtr or a valid IonScript*.
GCStructPtr<IonScript*> ionScript_;
// For functions that need a CallObject and/or NamedLambdaObject, the template
// objects used by the Baseline JIT and Ion. If the function needs both a
// named lambda object and a call object, the named lambda object template is
// linked via the call object's enclosing environment. This field is set the
// first time the Baseline JIT compiles this script.
mozilla::Maybe<HeapPtr<EnvironmentObject*>> templateEnv_;
// Analysis data computed lazily the first time this script is compiled or
// inlined by WarpBuilder.
mozilla::Maybe<bool> usesEnvironmentChain_;
// The size of this allocation.
Offset endOffset_ = 0;
struct Flags {
// True if this script entered Ion via OSR at a loop header.
bool hadIonOSR : 1;
};
Flags flags_ = {}; // Zero-initialize flags.
js::UniquePtr<InliningRoot> inliningRoot_;
#ifdef DEBUG
// If the last warp compilation invalidated because of TranspiledCacheIR
// bailouts, this is a hash of the ICScripts used in that compilation.
// When recompiling, we assert that the hash has changed.
mozilla::Maybe<mozilla::HashNumber> failedICHash_;
// To avoid pathological cases, we skip the check if we have purged
// stubs due to GC pressure.
bool hasPurgedStubs_ = false;
#endif
// Value of the warmup counter when the last IC stub was attached,
// used for Ion hints.
uint32_t warmUpCountAtLastICStub_ = 0;
ICScript icScript_;
// End of fields.
Offset endOffset() const { return endOffset_; }
public:
JitScript(JSScript* script, Offset fallbackStubsOffset, Offset endOffset,
const char* profileString);
~JitScript();
JSScript* owningScript() const { return owningScript_; }
[[nodiscard]] bool ensureHasCachedBaselineJitData(JSContext* cx,
HandleScript script);
[[nodiscard]] bool ensureHasCachedIonData(JSContext* cx, HandleScript script);
void setHadIonOSR() { flags_.hadIonOSR = true; }
bool hadIonOSR() const { return flags_.hadIonOSR; }
uint32_t numICEntries() const { return icScript_.numICEntries(); }
#ifdef DEBUG
bool hasActiveICScript() const;
#endif
void resetAllActiveFlags();
void ensureProfileString(JSContext* cx, JSScript* script);
const char* profileString() const {
MOZ_ASSERT(profileString_);
return profileString_;
}
static void Destroy(Zone* zone, JitScript* script);
static constexpr Offset offsetOfICEntries() { return sizeof(JitScript); }
static constexpr size_t offsetOfBaselineScript() {
return offsetof(JitScript, baselineScript_);
}
static constexpr size_t offsetOfIonScript() {
return offsetof(JitScript, ionScript_);
}
static constexpr size_t offsetOfICScript() {
return offsetof(JitScript, icScript_);
}
static constexpr size_t offsetOfWarmUpCount() {
return offsetOfICScript() + ICScript::offsetOfWarmUpCount();
}
uint32_t warmUpCount() const { return icScript_.warmUpCount_; }
void incWarmUpCount() { icScript_.warmUpCount_++; }
void resetWarmUpCount(uint32_t count);
void prepareForDestruction(Zone* zone);
void addSizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf, size_t* data,
size_t* allocSites) const;
ICEntry& icEntry(size_t index) { return icScript_.icEntry(index); }
ICFallbackStub* fallbackStub(size_t index) {
return icScript_.fallbackStub(index);
}
ICEntry* icEntryForStub(const ICFallbackStub* stub) {
return icScript_.icEntryForStub(stub);
}
ICFallbackStub* fallbackStubForICEntry(const ICEntry* entry) {
return icScript_.fallbackStubForICEntry(entry);
}
void trace(JSTracer* trc);
void traceWeak(JSTracer* trc);
void purgeStubs(JSScript* script, ICStubSpace& newStubSpace);
void purgeInactiveICScripts();
ICEntry& icEntryFromPCOffset(uint32_t pcOffset) {
return icScript_.icEntryFromPCOffset(pcOffset);
};
size_t allocBytes() const { return endOffset(); }
EnvironmentObject* templateEnvironment() const { return templateEnv_.ref(); }
bool usesEnvironmentChain() const { return *usesEnvironmentChain_; }
bool resetAllocSites(bool resetNurserySites, bool resetPretenuredSites);
void updateLastICStubCounter() { warmUpCountAtLastICStub_ = warmUpCount(); }
uint32_t warmUpCountAtLastICStub() const { return warmUpCountAtLastICStub_; }
private:
// Methods to set baselineScript_ to a BaselineScript*, nullptr, or
// BaselineDisabledScriptPtr.
void setBaselineScriptImpl(JSScript* script, BaselineScript* baselineScript);
void setBaselineScriptImpl(JS::GCContext* gcx, JSScript* script,
BaselineScript* baselineScript);
public:
// Methods for getting/setting/clearing a BaselineScript*.
bool hasBaselineScript() const {
bool res = baselineScript_ && baselineScript_ != BaselineDisabledScriptPtr;
MOZ_ASSERT_IF(!res, !hasIonScript());
return res;
}
BaselineScript* baselineScript() const {
MOZ_ASSERT(hasBaselineScript());
return baselineScript_;
}
void setBaselineScript(JSScript* script, BaselineScript* baselineScript) {
MOZ_ASSERT(!hasBaselineScript());
setBaselineScriptImpl(script, baselineScript);
MOZ_ASSERT(hasBaselineScript());
}
[[nodiscard]] BaselineScript* clearBaselineScript(JS::GCContext* gcx,
JSScript* script) {
BaselineScript* baseline = baselineScript();
setBaselineScriptImpl(gcx, script, nullptr);
return baseline;
}
private:
// Methods to set ionScript_ to an IonScript*, nullptr, or one of the special
// Ion{Disabled,Compiling}ScriptPtr values.
void setIonScriptImpl(JS::GCContext* gcx, JSScript* script,
IonScript* ionScript);
void setIonScriptImpl(JSScript* script, IonScript* ionScript);
// Helper that calls the passed function for the outer ICScript and for each
// inlined ICScript.
template <typename F>
void forEachICScript(const F& f);
template <typename F>
void forEachICScript(const F& f) const;
public:
// Methods for getting/setting/clearing an IonScript*.
bool hasIonScript() const {
bool res = ionScript_ && ionScript_ != IonDisabledScriptPtr &&
ionScript_ != IonCompilingScriptPtr;
MOZ_ASSERT_IF(res, baselineScript_);
return res;
}
IonScript* ionScript() const {
MOZ_ASSERT(hasIonScript());
return ionScript_;
}
void setIonScript(JSScript* script, IonScript* ionScript) {
MOZ_ASSERT(!hasIonScript());
setIonScriptImpl(script, ionScript);
MOZ_ASSERT(hasIonScript());
}
[[nodiscard]] IonScript* clearIonScript(JS::GCContext* gcx,
JSScript* script) {
IonScript* ion = ionScript();
setIonScriptImpl(gcx, script, nullptr);
return ion;
}
// Methods for off-thread compilation.
bool isIonCompilingOffThread() const {
return ionScript_ == IonCompilingScriptPtr;
}
void setIsIonCompilingOffThread(JSScript* script) {
MOZ_ASSERT(ionScript_ == nullptr);
setIonScriptImpl(script, IonCompilingScriptPtr);
}
void clearIsIonCompilingOffThread(JSScript* script) {
MOZ_ASSERT(isIonCompilingOffThread());
setIonScriptImpl(script, nullptr);
}
ICScript* icScript() { return &icScript_; }
bool hasInliningRoot() const { return !!inliningRoot_; }
InliningRoot* inliningRoot() const { return inliningRoot_.get(); }
InliningRoot* getOrCreateInliningRoot(JSContext* cx, JSScript* script);
inline void notePurgedStubs() {
#ifdef DEBUG
failedICHash_.reset();
hasPurgedStubs_ = true;
#endif
}
#ifdef DEBUG
bool hasPurgedStubs() const { return hasPurgedStubs_; }
bool hasFailedICHash() const { return failedICHash_.isSome(); }
mozilla::HashNumber getFailedICHash() { return failedICHash_.extract(); }
void setFailedICHash(mozilla::HashNumber hash) {
MOZ_ASSERT(failedICHash_.isNothing());
if (!hasPurgedStubs_) {
failedICHash_.emplace(hash);
}
}
#endif
};
// Ensures no JitScripts are purged in the current zone.
class MOZ_RAII AutoKeepJitScripts {
jit::JitZone* zone_;
bool prev_;
AutoKeepJitScripts(const AutoKeepJitScripts&) = delete;
void operator=(const AutoKeepJitScripts&) = delete;
public:
explicit inline AutoKeepJitScripts(JSContext* cx);
inline ~AutoKeepJitScripts();
};
// Mark ICScripts on the stack as active, so that they are not discarded
// during GC, and copy active Baseline IC stubs to the new stub space.
void MarkActiveICScriptsAndCopyStubs(Zone* zone, ICStubSpace& newStubSpace);
#ifdef JS_STRUCTURED_SPEW
void JitSpewBaselineICStats(JSScript* script, const char* dumpReason);
#endif
} // namespace jit
} // namespace js
#endif /* jit_JitScript_h */