Source code
Revision control
Copy as Markdown
Other Tools
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: set ts=8 sts=2 et sw=2 tw=80:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
#include "vm/HelperThreads.h"
#include "mozilla/ReverseIterator.h" // mozilla::Reversed(...)
#include "mozilla/ScopeExit.h"
#include "mozilla/Span.h" // mozilla::Span<TaggedScriptThingIndex>
#include <algorithm>
#include "frontend/CompilationStencil.h" // frontend::CompilationStencil
#include "gc/GC.h"
#include "gc/Zone.h"
#include "jit/Ion.h"
#include "jit/IonCompileTask.h"
#include "jit/JitRuntime.h"
#include "jit/JitScript.h"
#include "js/CompileOptions.h" // JS::PrefableCompileOptions, JS::ReadOnlyCompileOptions
#include "js/experimental/CompileScript.h" // JS::ThreadStackQuotaForSize
#include "js/friend/StackLimits.h" // js::ReportOverRecursed
#include "js/HelperThreadAPI.h"
#include "js/Stack.h"
#include "js/UniquePtr.h"
#include "js/Utility.h"
#include "threading/CpuCount.h"
#include "vm/ErrorReporting.h"
#include "vm/HelperThreadState.h"
#include "vm/InternalThreadPool.h"
#include "vm/MutexIDs.h"
#include "wasm/WasmGenerator.h"
using namespace js;
using mozilla::TimeDuration;
static void CancelOffThreadWasmCompleteTier2GeneratorLocked(
AutoLockHelperThreadState& lock);
static void CancelOffThreadWasmPartialTier2CompileLocked(
AutoLockHelperThreadState& lock);
// This file is structured as follows:
//
// (1) Methods for GlobalHelperThreadState, and top level scheduling logic
// (2) Specifics for JS task classes
// (3) Specifics for wasm task classes
///////////////////////////////////////////////////////////////////////////
// //
// GlobalHelperThreadState methods and top-level scheduling logic //
// //
///////////////////////////////////////////////////////////////////////////
namespace js {
MOZ_RUNINIT Mutex gHelperThreadLock(mutexid::GlobalHelperThreadState);
GlobalHelperThreadState* gHelperThreadState = nullptr;
} // namespace js
bool js::CreateHelperThreadsState() {
MOZ_ASSERT(!gHelperThreadState);
gHelperThreadState = js_new<GlobalHelperThreadState>();
return gHelperThreadState;
}
void js::DestroyHelperThreadsState() {
AutoLockHelperThreadState lock;
if (!gHelperThreadState) {
return;
}
gHelperThreadState->finish(lock);
js_delete(gHelperThreadState);
gHelperThreadState = nullptr;
}
bool js::EnsureHelperThreadsInitialized() {
MOZ_ASSERT(gHelperThreadState);
return gHelperThreadState->ensureInitialized();
}
static size_t ClampDefaultCPUCount(size_t cpuCount) {
// It's extremely rare for SpiderMonkey to have more than a few cores worth
// of work. At higher core counts, performance can even decrease due to NUMA
// (and SpiderMonkey's lack of NUMA-awareness), contention, and general lack
// of optimization for high core counts. So to avoid wasting thread stack
// resources (and cluttering gdb and core dumps), clamp to 8 cores for now.
return std::min<size_t>(cpuCount, 8);
}
static size_t ThreadCountForCPUCount(size_t cpuCount) {
// We need at least two threads for tier-2 wasm compilations, because
// there's a master task that holds a thread while other threads do the
// compilation.
return std::max<size_t>(cpuCount, 2);
}
bool js::SetFakeCPUCount(size_t count) {
HelperThreadState().setCpuCount(count);
return true;
}
void GlobalHelperThreadState::setCpuCount(size_t count) {
// This must be called before any threads have been initialized.
AutoLockHelperThreadState lock;
MOZ_ASSERT(!isInitialized(lock));
// We can't do this if an external thread pool is in use.
MOZ_ASSERT(!dispatchTaskCallback);
cpuCount = count;
threadCount = ThreadCountForCPUCount(count);
}
size_t js::GetHelperThreadCount() { return HelperThreadState().threadCount; }
size_t js::GetHelperThreadCPUCount() { return HelperThreadState().cpuCount; }
void JS::SetProfilingThreadCallbacks(
JS::RegisterThreadCallback registerThread,
JS::UnregisterThreadCallback unregisterThread) {
HelperThreadState().registerThread = registerThread;
HelperThreadState().unregisterThread = unregisterThread;
}
// error for HelperThreadTaskCallback.
JS_PUBLIC_API MOZ_NEVER_INLINE void JS::SetHelperThreadTaskCallback(
HelperThreadTaskCallback callback, size_t threadCount, size_t stackSize) {
AutoLockHelperThreadState lock;
HelperThreadState().setDispatchTaskCallback(callback, threadCount, stackSize,
lock);
}
JS_PUBLIC_API MOZ_NEVER_INLINE const char* JS::GetHelperThreadTaskName(
HelperThreadTask* task) {
return task->getName();
}
void GlobalHelperThreadState::setDispatchTaskCallback(
JS::HelperThreadTaskCallback callback, size_t threadCount, size_t stackSize,
const AutoLockHelperThreadState& lock) {
MOZ_ASSERT(!isInitialized(lock));
MOZ_ASSERT(!dispatchTaskCallback);
MOZ_ASSERT(threadCount != 0);
MOZ_ASSERT(stackSize >= 16 * 1024);
dispatchTaskCallback = callback;
this->threadCount = threadCount;
this->stackQuota = JS::ThreadStackQuotaForSize(stackSize);
}
bool GlobalHelperThreadState::ensureInitialized() {
MOZ_ASSERT(CanUseExtraThreads());
MOZ_ASSERT(this == &HelperThreadState());
AutoLockHelperThreadState lock;
if (isInitialized(lock)) {
return true;
}
for (size_t& i : runningTaskCount) {
i = 0;
}
useInternalThreadPool_ = !dispatchTaskCallback;
if (useInternalThreadPool(lock)) {
if (!InternalThreadPool::Initialize(threadCount, lock)) {
return false;
}
}
MOZ_ASSERT(dispatchTaskCallback);
if (!ensureThreadCount(threadCount, lock)) {
finishThreads(lock);
return false;
}
MOZ_ASSERT(threadCount != 0);
isInitialized_ = true;
return true;
}
bool GlobalHelperThreadState::ensureThreadCount(
size_t count, AutoLockHelperThreadState& lock) {
if (!helperTasks_.reserve(count)) {
return false;
}
if (useInternalThreadPool(lock)) {
InternalThreadPool& pool = InternalThreadPool::Get();
if (pool.threadCount(lock) < count) {
if (!pool.ensureThreadCount(count, lock)) {
return false;
}
threadCount = pool.threadCount(lock);
}
}
return true;
}
GlobalHelperThreadState::GlobalHelperThreadState()
: cpuCount(0),
threadCount(0),
totalCountRunningTasks(0),
registerThread(nullptr),
unregisterThread(nullptr),
wasmCompleteTier2GeneratorsFinished_(0) {
MOZ_ASSERT(!gHelperThreadState);
cpuCount = ClampDefaultCPUCount(GetCPUCount());
threadCount = ThreadCountForCPUCount(cpuCount);
MOZ_ASSERT(cpuCount > 0, "GetCPUCount() seems broken");
}
void GlobalHelperThreadState::finish(AutoLockHelperThreadState& lock) {
if (!isInitialized(lock)) {
return;
}
MOZ_ASSERT_IF(!JSRuntime::hasLiveRuntimes(), gcParallelMarkingThreads == 0);
finishThreads(lock);
// Make sure there are no Ion free tasks left. We check this here because,
// unlike the other tasks, we don't explicitly block on this when
// destroying a runtime.
auto& freeList = ionFreeList(lock);
while (!freeList.empty()) {
UniquePtr<jit::IonFreeTask> task = std::move(freeList.back());
freeList.popBack();
jit::FreeIonCompileTasks(task->compileTasks());
}
}
void GlobalHelperThreadState::finishThreads(AutoLockHelperThreadState& lock) {
waitForAllTasksLocked(lock);
terminating_ = true;
if (InternalThreadPool::IsInitialized()) {
InternalThreadPool::ShutDown(lock);
}
}
#ifdef DEBUG
void GlobalHelperThreadState::assertIsLockedByCurrentThread() const {
gHelperThreadLock.assertOwnedByCurrentThread();
}
#endif // DEBUG
void GlobalHelperThreadState::dispatch(const AutoLockHelperThreadState& lock) {
if (helperTasks_.length() >= threadCount) {
return;
}
HelperThreadTask* task = findHighestPriorityTask(lock);
if (!task) {
return;
}
#ifdef DEBUG
MOZ_ASSERT(tasksPending_ < threadCount);
tasksPending_++;
#endif
// Add task to list of running tasks immediately.
helperTasks(lock).infallibleEmplaceBack(task);
runningTaskCount[task->threadType()]++;
totalCountRunningTasks++;
lock.queueTaskToDispatch(task);
}
void GlobalHelperThreadState::wait(
AutoLockHelperThreadState& lock,
TimeDuration timeout /* = TimeDuration::Forever() */) {
MOZ_ASSERT(!lock.hasQueuedTasks());
consumerWakeup.wait_for(lock, timeout);
}
void GlobalHelperThreadState::notifyAll(const AutoLockHelperThreadState&) {
consumerWakeup.notify_all();
}
void GlobalHelperThreadState::notifyOne(const AutoLockHelperThreadState&) {
consumerWakeup.notify_one();
}
bool GlobalHelperThreadState::hasActiveThreads(
const AutoLockHelperThreadState& lock) {
return !helperTasks(lock).empty();
}
void js::WaitForAllHelperThreads() { HelperThreadState().waitForAllTasks(); }
void js::WaitForAllHelperThreads(AutoLockHelperThreadState& lock) {
HelperThreadState().waitForAllTasksLocked(lock);
}
void GlobalHelperThreadState::waitForAllTasks() {
AutoLockHelperThreadState lock;
waitForAllTasksLocked(lock);
}
void GlobalHelperThreadState::waitForAllTasksLocked(
AutoLockHelperThreadState& lock) {
CancelOffThreadWasmCompleteTier2GeneratorLocked(lock);
CancelOffThreadWasmPartialTier2CompileLocked(lock);
while (canStartTasks(lock) || hasActiveThreads(lock)) {
wait(lock);
}
MOZ_ASSERT(tasksPending_ == 0);
MOZ_ASSERT(gcParallelWorklist().isEmpty(lock));
MOZ_ASSERT(ionWorklist(lock).empty());
MOZ_ASSERT(wasmWorklist(lock, wasm::CompileState::EagerTier1).empty());
MOZ_ASSERT(promiseHelperTasks(lock).empty());
MOZ_ASSERT(compressionWorklist(lock).empty());
MOZ_ASSERT(ionFreeList(lock).empty());
MOZ_ASSERT(wasmWorklist(lock, wasm::CompileState::EagerTier2).empty());
MOZ_ASSERT(wasmCompleteTier2GeneratorWorklist(lock).empty());
MOZ_ASSERT(wasmPartialTier2CompileWorklist(lock).empty());
MOZ_ASSERT(!tasksPending_);
MOZ_ASSERT(!hasActiveThreads(lock));
}
// A task can be a "master" task, ie, it will block waiting for other worker
// threads that perform work on its behalf. If so it must not take the last
// available thread; there must always be at least one worker thread able to do
// the actual work. (Or the system may deadlock.)
//
// If a task is a master task it *must* pass isMaster=true here, or perform a
// similar calculation to avoid deadlock from starvation.
//
// isMaster should only be true if the thread calling checkTaskThreadLimit() is
// a helper thread.
//
// NOTE: Calling checkTaskThreadLimit() from a helper thread in the dynamic
// region after currentTask.emplace() and before currentTask.reset() may cause
// it to return a different result than if it is called outside that dynamic
// region, as the predicate inspects the values of the threads' currentTask
// members.
bool GlobalHelperThreadState::checkTaskThreadLimit(
ThreadType threadType, size_t maxThreads, bool isMaster,
const AutoLockHelperThreadState& lock) const {
MOZ_ASSERT(maxThreads >= 1);
MOZ_ASSERT(maxThreads <= threadCount);
// Check thread limit for this task kind.
size_t count = runningTaskCount[threadType];
if (count >= maxThreads) {
return false;
}
// Check overall idle thread count taking into account master threads. A
// master thread must not use the last idle thread or it will deadlock itself.
MOZ_ASSERT(threadCount >= totalCountRunningTasks);
size_t idleCount = threadCount - totalCountRunningTasks;
size_t idleRequired = isMaster ? 2 : 1;
return idleCount >= idleRequired;
}
static inline bool IsHelperThreadSimulatingOOM(js::ThreadType threadType) {
#if defined(DEBUG) || defined(JS_OOM_BREAKPOINT)
return js::oom::simulator.targetThread() == threadType;
#else
return false;
#endif
}
void GlobalHelperThreadState::addSizeOfIncludingThis(
JS::GlobalStats* stats, const AutoLockHelperThreadState& lock) const {
#ifdef DEBUG
assertIsLockedByCurrentThread();
#endif
mozilla::MallocSizeOf mallocSizeOf = stats->mallocSizeOf_;
JS::HelperThreadStats& htStats = stats->helperThread;
htStats.stateData += mallocSizeOf(this);
if (InternalThreadPool::IsInitialized()) {
htStats.stateData +=
InternalThreadPool::Get().sizeOfIncludingThis(mallocSizeOf, lock);
}
// Report memory used by various containers
htStats.stateData +=
ionWorklist_.sizeOfExcludingThis(mallocSizeOf) +
ionFinishedList_.sizeOfExcludingThis(mallocSizeOf) +
ionFreeList_.sizeOfExcludingThis(mallocSizeOf) +
wasmWorklist_tier1_.sizeOfExcludingThis(mallocSizeOf) +
wasmWorklist_tier2_.sizeOfExcludingThis(mallocSizeOf) +
wasmCompleteTier2GeneratorWorklist_.sizeOfExcludingThis(mallocSizeOf) +
wasmPartialTier2CompileWorklist_.sizeOfExcludingThis(mallocSizeOf) +
promiseHelperTasks_.sizeOfExcludingThis(mallocSizeOf) +
compressionPendingList_.sizeOfExcludingThis(mallocSizeOf) +
compressionWorklist_.sizeOfExcludingThis(mallocSizeOf) +
compressionFinishedList_.sizeOfExcludingThis(mallocSizeOf) +
gcParallelWorklist_.sizeOfExcludingThis(mallocSizeOf, lock) +
helperTasks_.sizeOfExcludingThis(mallocSizeOf);
// Report IonCompileTasks on wait lists
for (auto task : ionWorklist_) {
htStats.ionCompileTask += task->sizeOfExcludingThis(mallocSizeOf);
}
for (auto task : ionFinishedList_) {
htStats.ionCompileTask += task->sizeOfExcludingThis(mallocSizeOf);
}
for (const auto& task : ionFreeList_) {
for (auto* compileTask : task->compileTasks()) {
htStats.ionCompileTask += compileTask->sizeOfExcludingThis(mallocSizeOf);
}
}
// Report wasm::CompileTasks on wait lists
for (auto task : wasmWorklist_tier1_) {
htStats.wasmCompile += task->sizeOfExcludingThis(mallocSizeOf);
}
for (auto task : wasmWorklist_tier2_) {
htStats.wasmCompile += task->sizeOfExcludingThis(mallocSizeOf);
}
// Report number of helper threads.
MOZ_ASSERT(htStats.idleThreadCount == 0);
MOZ_ASSERT(threadCount >= totalCountRunningTasks);
htStats.activeThreadCount = totalCountRunningTasks;
htStats.idleThreadCount = threadCount - totalCountRunningTasks;
}
size_t GlobalHelperThreadState::maxIonCompilationThreads() const {
if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_ION)) {
return 1;
}
return threadCount;
}
size_t GlobalHelperThreadState::maxIonFreeThreads() const {
// IonFree tasks are low priority. Limit to one thread to help avoid jemalloc
// lock contention.
return 1;
}
size_t GlobalHelperThreadState::maxPromiseHelperThreads() const {
if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_PROMISE_TASK)) {
return 1;
}
return std::min(cpuCount, threadCount);
}
size_t GlobalHelperThreadState::maxDelazifyThreads() const {
if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_DELAZIFY)) {
return 1;
}
return std::min(cpuCount, threadCount);
}
size_t GlobalHelperThreadState::maxCompressionThreads() const {
if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_COMPRESS)) {
return 1;
}
// Compression is triggered on major GCs to compress ScriptSources. It is
// considered low priority work.
return 1;
}
size_t GlobalHelperThreadState::maxGCParallelThreads() const {
if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_GCPARALLEL)) {
return 1;
}
return threadCount;
}
size_t GlobalHelperThreadState::maxWasmCompilationThreads() const {
if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_WASM_COMPILE_TIER1) ||
IsHelperThreadSimulatingOOM(js::THREAD_TYPE_WASM_COMPILE_TIER2)) {
return 1;
}
return std::min(cpuCount, threadCount);
}
size_t js::GetMaxWasmCompilationThreads() {
return HelperThreadState().maxWasmCompilationThreads();
}
size_t GlobalHelperThreadState::maxWasmCompleteTier2GeneratorThreads() const {
return MaxCompleteTier2GeneratorTasks;
}
size_t GlobalHelperThreadState::maxWasmPartialTier2CompileThreads() const {
return MaxPartialTier2CompileTasks;
}
void GlobalHelperThreadState::trace(JSTracer* trc) {
{
AutoLockHelperThreadState lock;
#ifdef DEBUG
// Since we hold the helper thread lock here we must disable GCMarker's
// checking of the atom marking bitmap since that also relies on taking the
// lock.
GCMarker* marker = nullptr;
if (trc->isMarkingTracer()) {
marker = GCMarker::fromTracer(trc);
marker->setCheckAtomMarking(false);
}
auto reenableAtomMarkingCheck = mozilla::MakeScopeExit([marker] {
if (marker) {
marker->setCheckAtomMarking(true);
}
});
#endif
for (auto task : ionWorklist(lock)) {
task->alloc().lifoAlloc()->setReadWrite();
task->trace(trc);
task->alloc().lifoAlloc()->setReadOnly();
}
for (auto task : ionFinishedList(lock)) {
task->trace(trc);
}
for (auto* helper : helperTasks(lock)) {
if (helper->is<jit::IonCompileTask>()) {
jit::IonCompileTask* ionCompileTask = helper->as<jit::IonCompileTask>();
ionCompileTask->alloc().lifoAlloc()->setReadWrite();
ionCompileTask->trace(trc);
}
}
}
// The lazy link list is only accessed on the main thread, so trace it after
// releasing the lock.
JSRuntime* rt = trc->runtime();
if (auto* jitRuntime = rt->jitRuntime()) {
jit::IonCompileTask* task = jitRuntime->ionLazyLinkList(rt).getFirst();
while (task) {
task->trace(trc);
task = task->getNext();
}
}
}
// Definition of helper thread tasks.
//
// Priority is determined by the order they're listed here.
const GlobalHelperThreadState::Selector GlobalHelperThreadState::selectors[] = {
&GlobalHelperThreadState::maybeGetGCParallelTask,
&GlobalHelperThreadState::maybeGetIonCompileTask,
&GlobalHelperThreadState::maybeGetWasmTier1CompileTask,
&GlobalHelperThreadState::maybeGetPromiseHelperTask,
&GlobalHelperThreadState::maybeGetFreeDelazifyTask,
&GlobalHelperThreadState::maybeGetDelazifyTask,
&GlobalHelperThreadState::maybeGetCompressionTask,
&GlobalHelperThreadState::maybeGetLowPrioIonCompileTask,
&GlobalHelperThreadState::maybeGetIonFreeTask,
&GlobalHelperThreadState::maybeGetWasmPartialTier2CompileTask,
&GlobalHelperThreadState::maybeGetWasmTier2CompileTask,
&GlobalHelperThreadState::maybeGetWasmCompleteTier2GeneratorTask};
bool GlobalHelperThreadState::canStartTasks(
const AutoLockHelperThreadState& lock) {
return canStartGCParallelTask(lock) || canStartIonCompileTask(lock) ||
canStartWasmTier1CompileTask(lock) ||
canStartPromiseHelperTask(lock) || canStartFreeDelazifyTask(lock) ||
canStartDelazifyTask(lock) || canStartCompressionTask(lock) ||
canStartIonFreeTask(lock) || canStartWasmTier2CompileTask(lock) ||
canStartWasmCompleteTier2GeneratorTask(lock) ||
canStartWasmPartialTier2CompileTask(lock);
}
void JS::RunHelperThreadTask(HelperThreadTask* task) {
MOZ_ASSERT(task);
MOZ_ASSERT(CanUseExtraThreads());
AutoLockHelperThreadState lock;
if (!gHelperThreadState || HelperThreadState().isTerminating(lock)) {
return;
}
HelperThreadState().runOneTask(task, lock);
HelperThreadState().dispatch(lock);
}
void GlobalHelperThreadState::runOneTask(HelperThreadTask* task,
AutoLockHelperThreadState& lock) {
#ifdef DEBUG
MOZ_ASSERT(tasksPending_ > 0);
tasksPending_--;
#endif
runTaskLocked(task, lock);
notifyAll(lock);
}
HelperThreadTask* GlobalHelperThreadState::findHighestPriorityTask(
const AutoLockHelperThreadState& locked) {
// Return the highest priority task that is ready to start, or nullptr.
for (const auto& selector : selectors) {
if (auto* task = (this->*(selector))(locked)) {
return task;
}
}
return nullptr;
}
#ifdef DEBUG
static bool VectorHasTask(
const Vector<HelperThreadTask*, 0, SystemAllocPolicy>& tasks,
HelperThreadTask* task) {
for (HelperThreadTask* t : tasks) {
if (t == task) {
return true;
}
}
return false;
}
#endif
void GlobalHelperThreadState::runTaskLocked(HelperThreadTask* task,
AutoLockHelperThreadState& locked) {
ThreadType threadType = task->threadType();
MOZ_ASSERT(VectorHasTask(helperTasks(locked), task));
MOZ_ASSERT(totalCountRunningTasks != 0);
MOZ_ASSERT(runningTaskCount[threadType] != 0);
js::oom::SetThreadType(threadType);
{
JS::AutoSuppressGCAnalysis nogc;
task->runHelperThreadTask(locked);
}
js::oom::SetThreadType(js::THREAD_TYPE_NONE);
helperTasks(locked).eraseIfEqual(task);
totalCountRunningTasks--;
runningTaskCount[threadType]--;
}
void AutoHelperTaskQueue::queueTaskToDispatch(
JS::HelperThreadTask* task) const {
// This is marked const because it doesn't release the mutex.
task->onThreadPoolDispatch();
AutoEnterOOMUnsafeRegion oomUnsafe;
if (!tasksToDispatch.append(task)) {
oomUnsafe.crash("AutoLockHelperThreadState::queueTaskToDispatch");
}
}
void AutoHelperTaskQueue::dispatchQueuedTasks() {
// The hazard analysis can't tell that the callback doesn't GC.
JS::AutoSuppressGCAnalysis nogc;
for (size_t i = 0; i < tasksToDispatch.length(); i++) {
HelperThreadState().dispatchTaskCallback(tasksToDispatch[i]);
}
tasksToDispatch.clear();
}
///////////////////////////////////////////////////////////////////////////
// //
// JS task definitions //
// //
///////////////////////////////////////////////////////////////////////////
//== IonCompileTask and CompilationSelector ===============================
bool GlobalHelperThreadState::canStartIonCompileTask(
const AutoLockHelperThreadState& lock) {
return !ionWorklist(lock).empty() &&
checkTaskThreadLimit(THREAD_TYPE_ION, maxIonCompilationThreads(),
lock);
}
static bool IonCompileTaskHasHigherPriority(jit::IonCompileTask* first,
jit::IonCompileTask* second) {
// Return true if priority(first) > priority(second).
//
// This method can return whatever it wants, though it really ought to be a
// total order. The ordering is allowed to race (change on the fly), however.
// A higher warm-up counter indicates a higher priority.
jit::JitScript* firstJitScript = first->script()->jitScript();
jit::JitScript* secondJitScript = second->script()->jitScript();
return firstJitScript->warmUpCount() / first->script()->length() >
secondJitScript->warmUpCount() / second->script()->length();
}
jit::IonCompileTask* GlobalHelperThreadState::highestPriorityPendingIonCompile(
const AutoLockHelperThreadState& lock, bool checkExecutionStatus) {
auto& worklist = ionWorklist(lock);
MOZ_ASSERT(!worklist.empty());
// Get the highest priority IonCompileTask which has not started compilation
// yet.
size_t index = worklist.length();
for (size_t i = 0; i < worklist.length(); i++) {
if (checkExecutionStatus && !worklist[i]->isMainThreadRunningJS()) {
continue;
}
if (i < index ||
IonCompileTaskHasHigherPriority(worklist[i], worklist[index])) {
index = i;
}
}
if (index == worklist.length()) {
return nullptr;
}
jit::IonCompileTask* task = worklist[index];
worklist.erase(&worklist[index]);
return task;
}
HelperThreadTask* GlobalHelperThreadState::maybeGetIonCompileTask(
const AutoLockHelperThreadState& lock) {
if (!canStartIonCompileTask(lock)) {
return nullptr;
}
return highestPriorityPendingIonCompile(lock,
/* checkExecutionStatus */ true);
}
HelperThreadTask* GlobalHelperThreadState::maybeGetLowPrioIonCompileTask(
const AutoLockHelperThreadState& lock) {
if (!canStartIonCompileTask(lock)) {
return nullptr;
}
return highestPriorityPendingIonCompile(lock,
/* checkExecutionStatus */ false);
}
bool GlobalHelperThreadState::submitTask(
jit::IonCompileTask* task, const AutoLockHelperThreadState& locked) {
MOZ_ASSERT(isInitialized(locked));
if (!ionWorklist(locked).append(task)) {
return false;
}
// The build is moving off-thread. Freeze the LifoAlloc to prevent any
// unwanted mutations.
task->alloc().lifoAlloc()->setReadOnly();
dispatch(locked);
return true;
}
bool js::StartOffThreadIonCompile(jit::IonCompileTask* task,
const AutoLockHelperThreadState& lock) {
return HelperThreadState().submitTask(task, lock);
}
/*
* Move an IonCompilationTask for which compilation has either finished, failed,
* or been cancelled into the global finished compilation list. All off thread
* compilations which are started must eventually be finished.
*/
void js::FinishOffThreadIonCompile(jit::IonCompileTask* task,
const AutoLockHelperThreadState& lock) {
AutoEnterOOMUnsafeRegion oomUnsafe;
if (!HelperThreadState().ionFinishedList(lock).append(task)) {
oomUnsafe.crash("FinishOffThreadIonCompile");
}
task->script()
->runtimeFromAnyThread()
->jitRuntime()
->numFinishedOffThreadTasksRef(lock)++;
}
static JSRuntime* GetSelectorRuntime(const CompilationSelector& selector) {
struct Matcher {
JSRuntime* operator()(JSScript* script) {
return script->runtimeFromMainThread();
}
JSRuntime* operator()(Zone* zone) { return zone->runtimeFromMainThread(); }
JSRuntime* operator()(ZonesInState zbs) { return zbs.runtime; }
JSRuntime* operator()(JSRuntime* runtime) { return runtime; }
};
return selector.match(Matcher());
}
static bool IonCompileTaskMatches(const CompilationSelector& selector,
jit::IonCompileTask* task) {
struct TaskMatches {
jit::IonCompileTask* task_;
bool operator()(JSScript* script) { return script == task_->script(); }
bool operator()(Zone* zone) {
return zone == task_->script()->zoneFromAnyThread();
}
bool operator()(JSRuntime* runtime) {
return runtime == task_->script()->runtimeFromAnyThread();
}
bool operator()(ZonesInState zbs) {
return zbs.runtime == task_->script()->runtimeFromAnyThread() &&
zbs.state == task_->script()->zoneFromAnyThread()->gcState();
}
};
return selector.match(TaskMatches{task});
}
// If we're canceling Ion compilations for a zone/runtime, force a new
// IonFreeTask even if there are just a few tasks. This lets us free as much
// memory as possible.
static bool ShouldForceIonFreeTask(const CompilationSelector& selector) {
struct Matcher {
bool operator()(JSScript* script) { return false; }
bool operator()(Zone* zone) { return true; }
bool operator()(ZonesInState zbs) { return true; }
bool operator()(JSRuntime* runtime) { return true; }
};
return selector.match(Matcher());
}
void GlobalHelperThreadState::cancelOffThreadIonCompile(
const CompilationSelector& selector) {
jit::JitRuntime* jitRuntime = GetSelectorRuntime(selector)->jitRuntime();
MOZ_ASSERT(jitRuntime);
AutoStartIonFreeTask freeTask(jitRuntime, ShouldForceIonFreeTask(selector));
{
AutoLockHelperThreadState lock;
if (!isInitialized(lock)) {
return;
}
/* Cancel any pending entries for which processing hasn't started. */
GlobalHelperThreadState::IonCompileTaskVector& worklist = ionWorklist(lock);
for (size_t i = 0; i < worklist.length(); i++) {
jit::IonCompileTask* task = worklist[i];
if (IonCompileTaskMatches(selector, task)) {
// Once finished, tasks are added to a Linked list which is
// allocated with the IonCompileTask class. The IonCompileTask is
// allocated in the LifoAlloc so we need the LifoAlloc to be mutable.
worklist[i]->alloc().lifoAlloc()->setReadWrite();
FinishOffThreadIonCompile(task, lock);
remove(worklist, &i);
}
}
/* Wait for in progress entries to finish up. */
bool cancelled;
do {
cancelled = false;
for (auto* helper : helperTasks(lock)) {
if (!helper->is<jit::IonCompileTask>()) {
continue;
}
jit::IonCompileTask* ionCompileTask = helper->as<jit::IonCompileTask>();
if (IonCompileTaskMatches(selector, ionCompileTask)) {
ionCompileTask->alloc().lifoAlloc()->setReadWrite();
ionCompileTask->mirGen().cancel();
cancelled = true;
}
}
if (cancelled) {
wait(lock);
}
} while (cancelled);
/* Cancel code generation for any completed entries. */
GlobalHelperThreadState::IonCompileTaskVector& finished =
ionFinishedList(lock);
for (size_t i = 0; i < finished.length(); i++) {
jit::IonCompileTask* task = finished[i];
if (IonCompileTaskMatches(selector, task)) {
JSRuntime* rt = task->script()->runtimeFromAnyThread();
jitRuntime->numFinishedOffThreadTasksRef(lock)--;
jit::FinishOffThreadTask(rt, freeTask, task);
remove(finished, &i);
}
}
}
/* Cancel lazy linking for pending tasks (attached to the ionScript). */
JSRuntime* runtime = GetSelectorRuntime(selector);
jit::IonCompileTask* task = jitRuntime->ionLazyLinkList(runtime).getFirst();
while (task) {
jit::IonCompileTask* next = task->getNext();
if (IonCompileTaskMatches(selector, task)) {
jit::FinishOffThreadTask(runtime, freeTask, task);
}
task = next;
}
}
static bool JitDataStructuresExist(const CompilationSelector& selector) {
struct Matcher {
bool operator()(JSScript* script) { return !!script->zone()->jitZone(); }
bool operator()(Zone* zone) { return !!zone->jitZone(); }
bool operator()(ZonesInState zbs) { return zbs.runtime->hasJitRuntime(); }
bool operator()(JSRuntime* runtime) { return runtime->hasJitRuntime(); }
};
return selector.match(Matcher());
}
void js::CancelOffThreadIonCompile(const CompilationSelector& selector) {
if (!JitDataStructuresExist(selector)) {
return;
}
if (jit::IsPortableBaselineInterpreterEnabled()) {
return;
}
HelperThreadState().cancelOffThreadIonCompile(selector);
}
#ifdef DEBUG
bool GlobalHelperThreadState::hasOffThreadIonCompile(
Zone* zone, AutoLockHelperThreadState& lock) {
for (jit::IonCompileTask* task : ionWorklist(lock)) {
if (task->script()->zoneFromAnyThread() == zone) {
return true;
}
}
for (auto* helper : helperTasks(lock)) {
if (helper->is<jit::IonCompileTask>()) {
JSScript* script = helper->as<jit::IonCompileTask>()->script();
if (script->zoneFromAnyThread() == zone) {
return true;
}
}
}
for (jit::IonCompileTask* task : ionFinishedList(lock)) {
if (task->script()->zoneFromAnyThread() == zone) {
return true;
}
}
JSRuntime* rt = zone->runtimeFromMainThread();
if (rt->hasJitRuntime()) {
for (jit::IonCompileTask* task : rt->jitRuntime()->ionLazyLinkList(rt)) {
if (task->script()->zone() == zone) {
return true;
}
}
}
return false;
}
bool js::HasOffThreadIonCompile(Zone* zone) {
if (jit::IsPortableBaselineInterpreterEnabled()) {
return false;
}
AutoLockHelperThreadState lock;
if (!HelperThreadState().isInitialized(lock)) {
return false;
}
return HelperThreadState().hasOffThreadIonCompile(zone, lock);
}
#endif
//== IonFreeTask ==========================================================
bool GlobalHelperThreadState::canStartIonFreeTask(
const AutoLockHelperThreadState& lock) {
return !ionFreeList(lock).empty() &&
checkTaskThreadLimit(THREAD_TYPE_ION_FREE, maxIonFreeThreads(), lock);
}
HelperThreadTask* GlobalHelperThreadState::maybeGetIonFreeTask(
const AutoLockHelperThreadState& lock) {
if (!canStartIonFreeTask(lock)) {
return nullptr;
}
UniquePtr<jit::IonFreeTask> task = std::move(ionFreeList(lock).back());
ionFreeList(lock).popBack();
return task.release();
}
void jit::JitRuntime::maybeStartIonFreeTask(bool force) {
IonFreeCompileTasks& tasks = ionFreeTaskBatch_.ref();
if (tasks.empty()) {
return;
}
// Start an IonFreeTask if we have at least eight tasks. If |force| is true we
// always start an IonFreeTask.
if (!force) {
constexpr size_t MinBatchSize = 8;
static_assert(IonFreeCompileTasks::InlineLength >= MinBatchSize,
"Minimum batch size shouldn't require malloc");
if (tasks.length() < MinBatchSize) {
return;
}
}
auto freeTask = js::MakeUnique<jit::IonFreeTask>(std::move(tasks));
if (!freeTask) {
// Free compilation data on the main thread instead.