Source code
Revision control
Copy as Markdown
Other Tools
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: set ts=8 sts=2 et sw=2 tw=80:
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
#include "jit/MIR.h"
#include "mozilla/CheckedInt.h"
#include "mozilla/EndianUtils.h"
#include "mozilla/FloatingPoint.h"
#include "mozilla/MathAlgorithms.h"
#include "mozilla/Maybe.h"
#include "mozilla/ScopeExit.h"
#include <array>
#include <utility>
#include "jslibmath.h"
#include "jsmath.h"
#include "jsnum.h"
#include "builtin/RegExp.h"
#include "jit/AtomicOperations.h"
#include "jit/CompileInfo.h"
#include "jit/KnownClass.h"
#include "jit/MIRGraph.h"
#include "jit/RangeAnalysis.h"
#include "jit/VMFunctions.h"
#include "jit/WarpBuilderShared.h"
#include "js/Conversions.h"
#include "js/experimental/JitInfo.h" // JSJitInfo, JSTypedMethodJitInfo
#include "js/ScalarType.h" // js::Scalar::Type
#include "util/Text.h"
#include "util/Unicode.h"
#include "vm/Iteration.h" // js::NativeIterator
#include "vm/PlainObject.h" // js::PlainObject
#include "vm/Uint8Clamped.h"
#include "wasm/WasmCode.h"
#include "wasm/WasmFeatures.h" // for wasm::ReportSimdAnalysis
#include "vm/JSAtomUtils-inl.h" // TypeName
#include "wasm/WasmInstance-inl.h"
using namespace js;
using namespace js::jit;
using JS::ToInt32;
using mozilla::CheckedInt;
using mozilla::DebugOnly;
using mozilla::IsFloat32Representable;
using mozilla::IsPowerOfTwo;
using mozilla::Maybe;
using mozilla::NumbersAreIdentical;
NON_GC_POINTER_TYPE_ASSERTIONS_GENERATED
#ifdef DEBUG
size_t MUse::index() const { return consumer()->indexOf(this); }
#endif
template <size_t Op>
static void ConvertDefinitionToDouble(TempAllocator& alloc, MDefinition* def,
MInstruction* consumer) {
MInstruction* replace = MToDouble::New(alloc, def);
consumer->replaceOperand(Op, replace);
consumer->block()->insertBefore(consumer, replace);
}
template <size_t Arity, size_t Index>
static void ConvertOperandToDouble(MAryInstruction<Arity>* def,
TempAllocator& alloc) {
static_assert(Index < Arity);
auto* operand = def->getOperand(Index);
if (operand->type() == MIRType::Float32) {
ConvertDefinitionToDouble<Index>(alloc, operand, def);
}
}
template <size_t Arity, size_t... ISeq>
static void ConvertOperandsToDouble(MAryInstruction<Arity>* def,
TempAllocator& alloc,
std::index_sequence<ISeq...>) {
(ConvertOperandToDouble<Arity, ISeq>(def, alloc), ...);
}
template <size_t Arity>
static void ConvertOperandsToDouble(MAryInstruction<Arity>* def,
TempAllocator& alloc) {
ConvertOperandsToDouble<Arity>(def, alloc, std::make_index_sequence<Arity>{});
}
template <size_t Arity, size_t... ISeq>
static bool AllOperandsCanProduceFloat32(MAryInstruction<Arity>* def,
std::index_sequence<ISeq...>) {
return (def->getOperand(ISeq)->canProduceFloat32() && ...);
}
template <size_t Arity>
static bool AllOperandsCanProduceFloat32(MAryInstruction<Arity>* def) {
return AllOperandsCanProduceFloat32<Arity>(def,
std::make_index_sequence<Arity>{});
}
static bool CheckUsesAreFloat32Consumers(const MInstruction* ins) {
if (ins->isImplicitlyUsed()) {
return false;
}
bool allConsumerUses = true;
for (MUseDefIterator use(ins); allConsumerUses && use; use++) {
allConsumerUses &= use.def()->canConsumeFloat32(use.use());
}
return allConsumerUses;
}
#ifdef JS_JITSPEW
static const char* OpcodeName(MDefinition::Opcode op) {
static const char* const names[] = {
# define NAME(x) #x,
MIR_OPCODE_LIST(NAME)
# undef NAME
};
return names[unsigned(op)];
}
void MDefinition::PrintOpcodeName(GenericPrinter& out, Opcode op) {
const char* name = OpcodeName(op);
size_t len = strlen(name);
for (size_t i = 0; i < len; i++) {
out.printf("%c", unicode::ToLowerCase(name[i]));
}
}
uint32_t js::jit::GetMBasicBlockId(const MBasicBlock* block) {
return block->id();
}
#endif
static MConstant* EvaluateInt64ConstantOperands(TempAllocator& alloc,
MBinaryInstruction* ins) {
MDefinition* left = ins->getOperand(0);
MDefinition* right = ins->getOperand(1);
if (!left->isConstant() || !right->isConstant()) {
return nullptr;
}
MOZ_ASSERT(left->type() == MIRType::Int64);
MOZ_ASSERT(right->type() == MIRType::Int64);
int64_t lhs = left->toConstant()->toInt64();
int64_t rhs = right->toConstant()->toInt64();
int64_t ret;
switch (ins->op()) {
case MDefinition::Opcode::BitAnd:
ret = lhs & rhs;
break;
case MDefinition::Opcode::BitOr:
ret = lhs | rhs;
break;
case MDefinition::Opcode::BitXor:
ret = lhs ^ rhs;
break;
case MDefinition::Opcode::Lsh:
ret = lhs << (rhs & 0x3F);
break;
case MDefinition::Opcode::Rsh:
ret = lhs >> (rhs & 0x3F);
break;
case MDefinition::Opcode::Ursh:
ret = uint64_t(lhs) >> (uint64_t(rhs) & 0x3F);
break;
case MDefinition::Opcode::Add:
ret = lhs + rhs;
break;
case MDefinition::Opcode::Sub:
ret = lhs - rhs;
break;
case MDefinition::Opcode::Mul:
ret = lhs * rhs;
break;
case MDefinition::Opcode::Div:
if (rhs == 0) {
// Division by zero will trap at runtime.
return nullptr;
}
if (ins->toDiv()->isUnsigned()) {
ret = int64_t(uint64_t(lhs) / uint64_t(rhs));
} else if (lhs == INT64_MIN || rhs == -1) {
// Overflow will trap at runtime.
return nullptr;
} else {
ret = lhs / rhs;
}
break;
case MDefinition::Opcode::Mod:
if (rhs == 0) {
// Division by zero will trap at runtime.
return nullptr;
}
if (!ins->toMod()->isUnsigned() && (lhs < 0 || rhs < 0)) {
// Handle all negative values at runtime, for simplicity.
return nullptr;
}
ret = int64_t(uint64_t(lhs) % uint64_t(rhs));
break;
default:
MOZ_CRASH("NYI");
}
return MConstant::NewInt64(alloc, ret);
}
static MConstant* EvaluateConstantOperands(TempAllocator& alloc,
MBinaryInstruction* ins,
bool* ptypeChange = nullptr) {
MDefinition* left = ins->getOperand(0);
MDefinition* right = ins->getOperand(1);
MOZ_ASSERT(IsTypeRepresentableAsDouble(left->type()));
MOZ_ASSERT(IsTypeRepresentableAsDouble(right->type()));
if (!left->isConstant() || !right->isConstant()) {
return nullptr;
}
MConstant* lhs = left->toConstant();
MConstant* rhs = right->toConstant();
double ret = JS::GenericNaN();
switch (ins->op()) {
case MDefinition::Opcode::BitAnd:
ret = double(lhs->toInt32() & rhs->toInt32());
break;
case MDefinition::Opcode::BitOr:
ret = double(lhs->toInt32() | rhs->toInt32());
break;
case MDefinition::Opcode::BitXor:
ret = double(lhs->toInt32() ^ rhs->toInt32());
break;
case MDefinition::Opcode::Lsh:
ret = double(uint32_t(lhs->toInt32()) << (rhs->toInt32() & 0x1F));
break;
case MDefinition::Opcode::Rsh:
ret = double(lhs->toInt32() >> (rhs->toInt32() & 0x1F));
break;
case MDefinition::Opcode::Ursh:
ret = double(uint32_t(lhs->toInt32()) >> (rhs->toInt32() & 0x1F));
break;
case MDefinition::Opcode::Add:
ret = lhs->numberToDouble() + rhs->numberToDouble();
break;
case MDefinition::Opcode::Sub:
ret = lhs->numberToDouble() - rhs->numberToDouble();
break;
case MDefinition::Opcode::Mul:
ret = lhs->numberToDouble() * rhs->numberToDouble();
break;
case MDefinition::Opcode::Div:
if (ins->toDiv()->isUnsigned()) {
if (rhs->isInt32(0)) {
if (ins->toDiv()->trapOnError()) {
return nullptr;
}
ret = 0.0;
} else {
ret = double(uint32_t(lhs->toInt32()) / uint32_t(rhs->toInt32()));
}
} else {
ret = NumberDiv(lhs->numberToDouble(), rhs->numberToDouble());
}
break;
case MDefinition::Opcode::Mod:
if (ins->toMod()->isUnsigned()) {
if (rhs->isInt32(0)) {
if (ins->toMod()->trapOnError()) {
return nullptr;
}
ret = 0.0;
} else {
ret = double(uint32_t(lhs->toInt32()) % uint32_t(rhs->toInt32()));
}
} else {
ret = NumberMod(lhs->numberToDouble(), rhs->numberToDouble());
}
break;
default:
MOZ_CRASH("NYI");
}
if (ins->type() == MIRType::Float32) {
return MConstant::NewFloat32(alloc, float(ret));
}
if (ins->type() == MIRType::Double) {
return MConstant::New(alloc, DoubleValue(ret));
}
Value retVal;
retVal.setNumber(JS::CanonicalizeNaN(ret));
// If this was an int32 operation but the result isn't an int32 (for
// example, a division where the numerator isn't evenly divisible by the
// denominator), decline folding.
MOZ_ASSERT(ins->type() == MIRType::Int32);
if (!retVal.isInt32()) {
if (ptypeChange) {
*ptypeChange = true;
}
return nullptr;
}
return MConstant::New(alloc, retVal);
}
static MMul* EvaluateExactReciprocal(TempAllocator& alloc, MDiv* ins) {
// we should fold only when it is a floating point operation
if (!IsFloatingPointType(ins->type())) {
return nullptr;
}
MDefinition* left = ins->getOperand(0);
MDefinition* right = ins->getOperand(1);
if (!right->isConstant()) {
return nullptr;
}
int32_t num;
if (!mozilla::NumberIsInt32(right->toConstant()->numberToDouble(), &num)) {
return nullptr;
}
// check if rhs is a power of two
if (mozilla::Abs(num) & (mozilla::Abs(num) - 1)) {
return nullptr;
}
Value ret;
ret.setDouble(1.0 / double(num));
MConstant* foldedRhs;
if (ins->type() == MIRType::Float32) {
foldedRhs = MConstant::NewFloat32(alloc, ret.toDouble());
} else {
foldedRhs = MConstant::New(alloc, ret);
}
MOZ_ASSERT(foldedRhs->type() == ins->type());
ins->block()->insertBefore(ins, foldedRhs);
MMul* mul = MMul::New(alloc, left, foldedRhs, ins->type());
mul->setMustPreserveNaN(ins->mustPreserveNaN());
return mul;
}
#ifdef JS_JITSPEW
const char* MDefinition::opName() const { return OpcodeName(op()); }
void MDefinition::printName(GenericPrinter& out) const {
PrintOpcodeName(out, op());
out.printf("%u", id());
}
#endif
HashNumber MDefinition::valueHash() const {
HashNumber out = HashNumber(op());
for (size_t i = 0, e = numOperands(); i < e; i++) {
out = addU32ToHash(out, getOperand(i)->id());
}
if (MDefinition* dep = dependency()) {
out = addU32ToHash(out, dep->id());
}
return out;
}
HashNumber MNullaryInstruction::valueHash() const {
HashNumber hash = HashNumber(op());
if (MDefinition* dep = dependency()) {
hash = addU32ToHash(hash, dep->id());
}
MOZ_ASSERT(hash == MDefinition::valueHash());
return hash;
}
HashNumber MUnaryInstruction::valueHash() const {
HashNumber hash = HashNumber(op());
hash = addU32ToHash(hash, getOperand(0)->id());
if (MDefinition* dep = dependency()) {
hash = addU32ToHash(hash, dep->id());
}
MOZ_ASSERT(hash == MDefinition::valueHash());
return hash;
}
HashNumber MBinaryInstruction::valueHash() const {
HashNumber hash = HashNumber(op());
hash = addU32ToHash(hash, getOperand(0)->id());
hash = addU32ToHash(hash, getOperand(1)->id());
if (MDefinition* dep = dependency()) {
hash = addU32ToHash(hash, dep->id());
}
MOZ_ASSERT(hash == MDefinition::valueHash());
return hash;
}
HashNumber MTernaryInstruction::valueHash() const {
HashNumber hash = HashNumber(op());
hash = addU32ToHash(hash, getOperand(0)->id());
hash = addU32ToHash(hash, getOperand(1)->id());
hash = addU32ToHash(hash, getOperand(2)->id());
if (MDefinition* dep = dependency()) {
hash = addU32ToHash(hash, dep->id());
}
MOZ_ASSERT(hash == MDefinition::valueHash());
return hash;
}
HashNumber MQuaternaryInstruction::valueHash() const {
HashNumber hash = HashNumber(op());
hash = addU32ToHash(hash, getOperand(0)->id());
hash = addU32ToHash(hash, getOperand(1)->id());
hash = addU32ToHash(hash, getOperand(2)->id());
hash = addU32ToHash(hash, getOperand(3)->id());
if (MDefinition* dep = dependency()) {
hash = addU32ToHash(hash, dep->id());
}
MOZ_ASSERT(hash == MDefinition::valueHash());
return hash;
}
const MDefinition* MDefinition::skipObjectGuards() const {
const MDefinition* result = this;
// These instructions don't modify the object and just guard specific
// properties.
while (true) {
if (result->isGuardShape()) {
result = result->toGuardShape()->object();
continue;
}
if (result->isGuardNullProto()) {
result = result->toGuardNullProto()->object();
continue;
}
if (result->isGuardProto()) {
result = result->toGuardProto()->object();
continue;
}
break;
}
return result;
}
bool MDefinition::congruentIfOperandsEqual(const MDefinition* ins) const {
if (op() != ins->op()) {
return false;
}
if (type() != ins->type()) {
return false;
}
if (isEffectful() || ins->isEffectful()) {
return false;
}
if (numOperands() != ins->numOperands()) {
return false;
}
for (size_t i = 0, e = numOperands(); i < e; i++) {
if (getOperand(i) != ins->getOperand(i)) {
return false;
}
}
return true;
}
MDefinition* MDefinition::foldsTo(TempAllocator& alloc) {
// In the default case, there are no constants to fold.
return this;
}
bool MDefinition::mightBeMagicType() const {
if (IsMagicType(type())) {
return true;
}
if (MIRType::Value != type()) {
return false;
}
return true;
}
bool MDefinition::definitelyType(std::initializer_list<MIRType> types) const {
#ifdef DEBUG
// Only support specialized, non-magic types.
auto isSpecializedNonMagic = [](MIRType type) {
return type <= MIRType::Object;
};
#endif
MOZ_ASSERT(types.size() > 0);
MOZ_ASSERT(std::all_of(types.begin(), types.end(), isSpecializedNonMagic));
if (type() == MIRType::Value) {
return false;
}
return std::find(types.begin(), types.end(), type()) != types.end();
}
MDefinition* MInstruction::foldsToStore(TempAllocator& alloc) {
if (!dependency()) {
return nullptr;
}
MDefinition* store = dependency();
if (mightAlias(store) != AliasType::MustAlias) {
return nullptr;
}
if (!store->block()->dominates(block())) {
return nullptr;
}
MDefinition* value;
switch (store->op()) {
case Opcode::StoreFixedSlot:
value = store->toStoreFixedSlot()->value();
break;
case Opcode::StoreDynamicSlot:
value = store->toStoreDynamicSlot()->value();
break;
case Opcode::StoreElement:
value = store->toStoreElement()->value();
break;
default:
MOZ_CRASH("unknown store");
}
// If the type are matching then we return the value which is used as
// argument of the store.
if (value->type() != type()) {
// If we expect to read a type which is more generic than the type seen
// by the store, then we box the value used by the store.
if (type() != MIRType::Value) {
return nullptr;
}
MOZ_ASSERT(value->type() < MIRType::Value);
MBox* box = MBox::New(alloc, value);
value = box;
}
return value;
}
void MDefinition::analyzeEdgeCasesForward() {}
void MDefinition::analyzeEdgeCasesBackward() {}
void MInstruction::setResumePoint(MResumePoint* resumePoint) {
MOZ_ASSERT(!resumePoint_);
resumePoint_ = resumePoint;
resumePoint_->setInstruction(this);
}
void MInstruction::stealResumePoint(MInstruction* other) {
MResumePoint* resumePoint = other->resumePoint_;
other->resumePoint_ = nullptr;
resumePoint->resetInstruction();
setResumePoint(resumePoint);
}
void MInstruction::moveResumePointAsEntry() {
MOZ_ASSERT(isNop());
block()->clearEntryResumePoint();
block()->setEntryResumePoint(resumePoint_);
resumePoint_->resetInstruction();
resumePoint_ = nullptr;
}
void MInstruction::clearResumePoint() {
resumePoint_->resetInstruction();
block()->discardPreAllocatedResumePoint(resumePoint_);
resumePoint_ = nullptr;
}
MDefinition* MTest::foldsDoubleNegation(TempAllocator& alloc) {
MDefinition* op = getOperand(0);
if (op->isNot()) {
// If the operand of the Not is itself a Not, they cancel out.
MDefinition* opop = op->getOperand(0);
if (opop->isNot()) {
return MTest::New(alloc, opop->toNot()->input(), ifTrue(), ifFalse());
}
return MTest::New(alloc, op->toNot()->input(), ifFalse(), ifTrue());
}
return nullptr;
}
MDefinition* MTest::foldsConstant(TempAllocator& alloc) {
MDefinition* op = getOperand(0);
if (MConstant* opConst = op->maybeConstantValue()) {
bool b;
if (opConst->valueToBoolean(&b)) {
return MGoto::New(alloc, b ? ifTrue() : ifFalse());
}
}
return nullptr;
}
MDefinition* MTest::foldsTypes(TempAllocator& alloc) {
MDefinition* op = getOperand(0);
switch (op->type()) {
case MIRType::Undefined:
case MIRType::Null:
return MGoto::New(alloc, ifFalse());
case MIRType::Symbol:
return MGoto::New(alloc, ifTrue());
default:
break;
}
return nullptr;
}
class UsesIterator {
MDefinition* def_;
public:
explicit UsesIterator(MDefinition* def) : def_(def) {}
auto begin() const { return def_->usesBegin(); }
auto end() const { return def_->usesEnd(); }
};
static bool AllInstructionsDeadIfUnused(MBasicBlock* block) {
for (auto* ins : *block) {
// Skip trivial instructions.
if (ins->isNop() || ins->isGoto()) {
continue;
}
// All uses must be within the current block.
for (auto* use : UsesIterator(ins)) {
if (use->consumer()->block() != block) {
return false;
}
}
// All instructions within this block must be dead if unused.
if (!DeadIfUnused(ins)) {
return false;
}
}
return true;
}
MDefinition* MTest::foldsNeedlessControlFlow(TempAllocator& alloc) {
// All instructions within both successors need be dead if unused.
if (!AllInstructionsDeadIfUnused(ifTrue()) ||
!AllInstructionsDeadIfUnused(ifFalse())) {
return nullptr;
}
// Both successors must have the same target successor.
if (ifTrue()->numSuccessors() != 1 || ifFalse()->numSuccessors() != 1) {
return nullptr;
}
if (ifTrue()->getSuccessor(0) != ifFalse()->getSuccessor(0)) {
return nullptr;
}
// The target successor's phis must be redundant. Redundant phis should have
// been removed in an earlier pass, so only check if any phis are present,
// which is a stronger condition.
if (ifTrue()->successorWithPhis()) {
return nullptr;
}
return MGoto::New(alloc, ifTrue());
}
MDefinition* MTest::foldsTo(TempAllocator& alloc) {
if (MDefinition* def = foldsDoubleNegation(alloc)) {
return def;
}
if (MDefinition* def = foldsConstant(alloc)) {
return def;
}
if (MDefinition* def = foldsTypes(alloc)) {
return def;
}
if (MDefinition* def = foldsNeedlessControlFlow(alloc)) {
return def;
}
return this;
}
AliasSet MThrow::getAliasSet() const {
return AliasSet::Store(AliasSet::ExceptionState);
}
AliasSet MNewArrayDynamicLength::getAliasSet() const {
return AliasSet::Store(AliasSet::ExceptionState);
}
AliasSet MNewTypedArrayDynamicLength::getAliasSet() const {
return AliasSet::Store(AliasSet::ExceptionState);
}
#ifdef JS_JITSPEW
void MDefinition::printOpcode(GenericPrinter& out) const {
PrintOpcodeName(out, op());
for (size_t j = 0, e = numOperands(); j < e; j++) {
out.printf(" ");
if (getUseFor(j)->hasProducer()) {
getOperand(j)->printName(out);
out.printf(":%s", StringFromMIRType(getOperand(j)->type()));
} else {
out.printf("(null)");
}
}
}
void MDefinition::dump(GenericPrinter& out) const {
printName(out);
out.printf(":%s", StringFromMIRType(type()));
out.printf(" = ");
printOpcode(out);
out.printf("\n");
if (isInstruction()) {
if (MResumePoint* resume = toInstruction()->resumePoint()) {
resume->dump(out);
}
}
}
void MDefinition::dump() const {
Fprinter out(stderr);
dump(out);
out.finish();
}
void MDefinition::dumpLocation(GenericPrinter& out) const {
MResumePoint* rp = nullptr;
const char* linkWord = nullptr;
if (isInstruction() && toInstruction()->resumePoint()) {
rp = toInstruction()->resumePoint();
linkWord = "at";
} else {
rp = block()->entryResumePoint();
linkWord = "after";
}
while (rp) {
JSScript* script = rp->block()->info().script();
uint32_t lineno = PCToLineNumber(rp->block()->info().script(), rp->pc());
out.printf(" %s %s:%u\n", linkWord, script->filename(), lineno);
rp = rp->caller();
linkWord = "in";
}
}
void MDefinition::dumpLocation() const {
Fprinter out(stderr);
dumpLocation(out);
out.finish();
}
#endif
#if defined(DEBUG) || defined(JS_JITSPEW)
size_t MDefinition::useCount() const {
size_t count = 0;
for (MUseIterator i(uses_.begin()); i != uses_.end(); i++) {
count++;
}
return count;
}
size_t MDefinition::defUseCount() const {
size_t count = 0;
for (MUseIterator i(uses_.begin()); i != uses_.end(); i++) {
if ((*i)->consumer()->isDefinition()) {
count++;
}
}
return count;
}
#endif
bool MDefinition::hasOneUse() const {
MUseIterator i(uses_.begin());
if (i == uses_.end()) {
return false;
}
i++;
return i == uses_.end();
}
bool MDefinition::hasOneDefUse() const {
bool hasOneDefUse = false;
for (MUseIterator i(uses_.begin()); i != uses_.end(); i++) {
if (!(*i)->consumer()->isDefinition()) {
continue;
}
// We already have a definition use. So 1+
if (hasOneDefUse) {
return false;
}
// We saw one definition. Loop to test if there is another.
hasOneDefUse = true;
}
return hasOneDefUse;
}
bool MDefinition::hasDefUses() const {
for (MUseIterator i(uses_.begin()); i != uses_.end(); i++) {
if ((*i)->consumer()->isDefinition()) {
return true;
}
}
return false;
}
bool MDefinition::hasLiveDefUses() const {
for (MUseIterator i(uses_.begin()); i != uses_.end(); i++) {
MNode* ins = (*i)->consumer();
if (ins->isDefinition()) {
if (!ins->toDefinition()->isRecoveredOnBailout()) {
return true;
}
} else {
MOZ_ASSERT(ins->isResumePoint());
if (!ins->toResumePoint()->isRecoverableOperand(*i)) {
return true;
}
}
}
return false;
}
MDefinition* MDefinition::maybeSingleDefUse() const {
MUseDefIterator use(this);
if (!use) {
// No def-uses.
return nullptr;
}
MDefinition* useDef = use.def();
use++;
if (use) {
// More than one def-use.
return nullptr;
}
return useDef;
}
MDefinition* MDefinition::maybeMostRecentlyAddedDefUse() const {
MUseDefIterator use(this);
if (!use) {
// No def-uses.
return nullptr;
}
MDefinition* mostRecentUse = use.def();
#ifdef DEBUG
// This function relies on addUse adding new uses to the front of the list.
// Check this invariant by asserting the next few uses are 'older'. Skip this
// for phis because setBackedge can add a new use for a loop phi even if the
// loop body has a use with an id greater than the loop phi's id.
if (!mostRecentUse->isPhi()) {
static constexpr size_t NumUsesToCheck = 3;
use++;
for (size_t i = 0; use && i < NumUsesToCheck; i++, use++) {
MOZ_ASSERT(use.def()->id() <= mostRecentUse->id());
}
}
#endif
return mostRecentUse;
}
void MDefinition::replaceAllUsesWith(MDefinition* dom) {
for (size_t i = 0, e = numOperands(); i < e; ++i) {
getOperand(i)->setImplicitlyUsedUnchecked();
}
justReplaceAllUsesWith(dom);
}
void MDefinition::justReplaceAllUsesWith(MDefinition* dom) {
MOZ_ASSERT(dom != nullptr);
MOZ_ASSERT(dom != this);
// Carry over the fact the value has uses which are no longer inspectable
// with the graph.
if (isImplicitlyUsed()) {
dom->setImplicitlyUsedUnchecked();
}
for (MUseIterator i(usesBegin()), e(usesEnd()); i != e; ++i) {
i->setProducerUnchecked(dom);
}
dom->uses_.takeElements(uses_);
}
bool MDefinition::optimizeOutAllUses(TempAllocator& alloc) {
for (MUseIterator i(usesBegin()), e(usesEnd()); i != e;) {
MUse* use = *i++;
MConstant* constant = use->consumer()->block()->optimizedOutConstant(alloc);
if (!alloc.ensureBallast()) {
return false;
}
// Update the resume point operand to use the optimized-out constant.
use->setProducerUnchecked(constant);
constant->addUseUnchecked(use);
}
// Remove dangling pointers.
this->uses_.clear();
return true;
}
void MDefinition::replaceAllLiveUsesWith(MDefinition* dom) {
for (MUseIterator i(usesBegin()), e(usesEnd()); i != e;) {
MUse* use = *i++;
MNode* consumer = use->consumer();
if (consumer->isResumePoint()) {
continue;
}
if (consumer->isDefinition() &&
consumer->toDefinition()->isRecoveredOnBailout()) {
continue;
}
// Update the operand to use the dominating definition.
use->replaceProducer(dom);
}
}
MConstant* MConstant::New(TempAllocator& alloc, const Value& v) {
return new (alloc) MConstant(alloc, v);
}
MConstant* MConstant::New(TempAllocator::Fallible alloc, const Value& v) {
return new (alloc) MConstant(alloc.alloc, v);
}
MConstant* MConstant::NewFloat32(TempAllocator& alloc, double d) {
MOZ_ASSERT(std::isnan(d) || d == double(float(d)));
return new (alloc) MConstant(float(d));
}
MConstant* MConstant::NewInt64(TempAllocator& alloc, int64_t i) {
return new (alloc) MConstant(MIRType::Int64, i);
}
MConstant* MConstant::NewIntPtr(TempAllocator& alloc, intptr_t i) {
return new (alloc) MConstant(MIRType::IntPtr, i);
}
MConstant* MConstant::New(TempAllocator& alloc, const Value& v, MIRType type) {
if (type == MIRType::Float32) {
return NewFloat32(alloc, v.toNumber());
}
MConstant* res = New(alloc, v);
MOZ_ASSERT(res->type() == type);
return res;
}
MConstant* MConstant::NewObject(TempAllocator& alloc, JSObject* v) {
return new (alloc) MConstant(v);
}
MConstant* MConstant::NewShape(TempAllocator& alloc, Shape* s) {
return new (alloc) MConstant(s);
}
static MIRType MIRTypeFromValue(const js::Value& vp) {
if (vp.isDouble()) {
return MIRType::Double;
}
if (vp.isMagic()) {
switch (vp.whyMagic()) {
case JS_OPTIMIZED_OUT:
return MIRType::MagicOptimizedOut;
case JS_ELEMENTS_HOLE:
return MIRType::MagicHole;
case JS_IS_CONSTRUCTING:
return MIRType::MagicIsConstructing;
case JS_UNINITIALIZED_LEXICAL:
return MIRType::MagicUninitializedLexical;
default:
MOZ_ASSERT_UNREACHABLE("Unexpected magic constant");
}
}
return MIRTypeFromValueType(vp.extractNonDoubleType());
}
MConstant::MConstant(TempAllocator& alloc, const js::Value& vp)
: MNullaryInstruction(classOpcode) {
setResultType(MIRTypeFromValue(vp));
MOZ_ASSERT(payload_.asBits == 0);
switch (type()) {
case MIRType::Undefined:
case MIRType::Null:
break;
case MIRType::Boolean:
payload_.b = vp.toBoolean();
break;
case MIRType::Int32:
payload_.i32 = vp.toInt32();
break;
case MIRType::Double:
payload_.d = vp.toDouble();
break;
case MIRType::String:
MOZ_ASSERT(!IsInsideNursery(vp.toString()));
MOZ_ASSERT(vp.toString()->isLinear());
payload_.str = vp.toString();
break;
case MIRType::Symbol:
payload_.sym = vp.toSymbol();
break;
case MIRType::BigInt:
MOZ_ASSERT(!IsInsideNursery(vp.toBigInt()));
payload_.bi = vp.toBigInt();
break;
case MIRType::Object:
MOZ_ASSERT(!IsInsideNursery(&vp.toObject()));
payload_.obj = &vp.toObject();
break;
case MIRType::MagicOptimizedOut:
case MIRType::MagicHole:
case MIRType::MagicIsConstructing:
case MIRType::MagicUninitializedLexical:
break;
default:
MOZ_CRASH("Unexpected type");
}
setMovable();
}
MConstant::MConstant(JSObject* obj) : MNullaryInstruction(classOpcode) {
MOZ_ASSERT(!IsInsideNursery(obj));
setResultType(MIRType::Object);
payload_.obj = obj;
setMovable();
}
MConstant::MConstant(Shape* shape) : MNullaryInstruction(classOpcode) {
setResultType(MIRType::Shape);
payload_.shape = shape;
setMovable();
}
MConstant::MConstant(float f) : MNullaryInstruction(classOpcode) {
setResultType(MIRType::Float32);
payload_.f = f;
setMovable();
}
MConstant::MConstant(MIRType type, int64_t i)
: MNullaryInstruction(classOpcode) {
MOZ_ASSERT(type == MIRType::Int64 || type == MIRType::IntPtr);
setResultType(type);
if (type == MIRType::Int64) {
payload_.i64 = i;
} else {
payload_.iptr = i;
}
setMovable();
}
#ifdef DEBUG
void MConstant::assertInitializedPayload() const {
// valueHash() and equals() expect the unused payload bits to be
// initialized to zero. Assert this in debug builds.
switch (type()) {
case MIRType::Int32:
case MIRType::Float32:
# if MOZ_LITTLE_ENDIAN()
MOZ_ASSERT((payload_.asBits >> 32) == 0);
# else
MOZ_ASSERT((payload_.asBits << 32) == 0);
# endif
break;
case MIRType::Boolean:
# if MOZ_LITTLE_ENDIAN()
MOZ_ASSERT((payload_.asBits >> 1) == 0);
# else
MOZ_ASSERT((payload_.asBits & ~(1ULL << 56)) == 0);
# endif
break;
case MIRType::Double:
case MIRType::Int64:
break;
case MIRType::String:
case MIRType::Object:
case MIRType::Symbol:
case MIRType::BigInt:
case MIRType::IntPtr:
case MIRType::Shape:
# if MOZ_LITTLE_ENDIAN()
MOZ_ASSERT_IF(JS_BITS_PER_WORD == 32, (payload_.asBits >> 32) == 0);
# else
MOZ_ASSERT_IF(JS_BITS_PER_WORD == 32, (payload_.asBits << 32) == 0);
# endif
break;
default:
MOZ_ASSERT(IsNullOrUndefined(type()) || IsMagicType(type()));
MOZ_ASSERT(payload_.asBits == 0);
break;
}
}
#endif
static HashNumber ConstantValueHash(MIRType type, uint64_t payload) {
// Build a 64-bit value holding both the payload and the type.
static const size_t TypeBits = 8;
static const size_t TypeShift = 64 - TypeBits;
MOZ_ASSERT(uintptr_t(type) <= (1 << TypeBits) - 1);
uint64_t bits = (uint64_t(type) << TypeShift) ^ payload;
// Fold all 64 bits into the 32-bit result. It's tempting to just discard
// half of the bits, as this is just a hash, however there are many common
// patterns of values where only the low or the high bits vary, so
// discarding either side would lead to excessive hash collisions.
return (HashNumber)bits ^ (HashNumber)(bits >> 32);
}
HashNumber MConstant::valueHash() const {
static_assert(sizeof(Payload) == sizeof(uint64_t),
"Code below assumes payload fits in 64 bits");
assertInitializedPayload();
return ConstantValueHash(type(), payload_.asBits);
}
HashNumber MConstantProto::valueHash() const {
HashNumber hash = protoObject()->valueHash();
const MDefinition* receiverObject = getReceiverObject();
if (receiverObject) {
hash = addU32ToHash(hash, receiverObject->id());
}
return hash;
}
bool MConstant::congruentTo(const MDefinition* ins) const {
return ins->isConstant() && equals(ins->toConstant());
}
#ifdef JS_JITSPEW
void MConstant::printOpcode(GenericPrinter& out) const {
PrintOpcodeName(out, op());
out.printf(" ");
switch (type()) {
case MIRType::Undefined:
out.printf("undefined");
break;
case MIRType::Null:
out.printf("null");
break;
case MIRType::Boolean:
out.printf(toBoolean() ? "true" : "false");
break;
case MIRType::Int32:
out.printf("0x%x", uint32_t(toInt32()));
break;
case MIRType::Int64:
out.printf("0x%" PRIx64, uint64_t(toInt64()));
break;
case MIRType::IntPtr:
out.printf("0x%" PRIxPTR, uintptr_t(toIntPtr()));
break;
case MIRType::Double:
out.printf("%.16g", toDouble());
break;
case MIRType::Float32: {
float val = toFloat32();
out.printf("%.16g", val);
break;
}
case MIRType::Object:
if (toObject().is<JSFunction>()) {
JSFunction* fun = &toObject().as<JSFunction>();
if (fun->displayAtom()) {
out.put("function ");
EscapedStringPrinter(out, fun->displayAtom(), 0);
} else {
out.put("unnamed function");
}
if (fun->hasBaseScript()) {
BaseScript* script = fun->baseScript();
out.printf(" (%s:%u)", script->filename() ? script->filename() : "",
script->lineno());
}
out.printf(" at %p", (void*)fun);
break;
}
out.printf("object %p (%s)", (void*)&toObject(),
toObject().getClass()->name);
break;
case MIRType::Symbol:
out.printf("symbol at %p", (void*)toSymbol());
break;
case MIRType::BigInt:
out.printf("BigInt at %p", (void*)toBigInt());
break;
case MIRType::String:
out.printf("string %p", (void*)toString());
break;
case MIRType::Shape:
out.printf("shape at %p", (void*)toShape());
break;
case MIRType::MagicHole:
out.printf("magic hole");
break;
case MIRType::MagicIsConstructing:
out.printf("magic is-constructing");
break;
case MIRType::MagicOptimizedOut:
out.printf("magic optimized-out");
break;
case MIRType::MagicUninitializedLexical:
out.printf("magic uninitialized-lexical");
break;
default:
MOZ_CRASH("unexpected type");
}
}
#endif
bool MConstant::canProduceFloat32() const {
if (!isTypeRepresentableAsDouble()) {
return false;
}
if (type() == MIRType::Int32) {
return IsFloat32Representable(static_cast<double>(toInt32()));
}
if (type() == MIRType::Double) {
return IsFloat32Representable(toDouble());
}
MOZ_ASSERT(type() == MIRType::Float32);
return true;
}
Value MConstant::toJSValue() const {
// Wasm has types like int64 that cannot be stored as js::Value. It also
// doesn't want the NaN canonicalization enforced by js::Value.
MOZ_ASSERT(!IsCompilingWasm());
switch (type()) {
case MIRType::Undefined:
return UndefinedValue();
case MIRType::Null:
return NullValue();
case MIRType::Boolean:
return BooleanValue(toBoolean());
case MIRType::Int32:
return Int32Value(toInt32());
case MIRType::Double:
return DoubleValue(toDouble());
case MIRType::Float32:
return Float32Value(toFloat32());
case MIRType::String:
return StringValue(toString());
case MIRType::Symbol:
return SymbolValue(toSymbol());
case MIRType::BigInt:
return BigIntValue(toBigInt());
case MIRType::Object:
return ObjectValue(toObject());
case MIRType::Shape:
return PrivateGCThingValue(toShape());
case MIRType::MagicOptimizedOut:
return MagicValue(JS_OPTIMIZED_OUT);
case MIRType::MagicHole:
return MagicValue(JS_ELEMENTS_HOLE);
case MIRType::MagicIsConstructing:
return MagicValue(JS_IS_CONSTRUCTING);
case MIRType::MagicUninitializedLexical:
return MagicValue(JS_UNINITIALIZED_LEXICAL);
default:
MOZ_CRASH("Unexpected type");
}
}
bool MConstant::valueToBoolean(bool* res) const {
switch (type()) {
case MIRType::Boolean:
*res = toBoolean();
return true;
case MIRType::Int32:
*res = toInt32() != 0;
return true;
case MIRType::Int64:
*res = toInt64() != 0;
return true;
case MIRType::Double:
*res = !std::isnan(toDouble()) && toDouble() != 0.0;
return true;
case MIRType::Float32:
*res = !std::isnan(toFloat32()) && toFloat32() != 0.0f;
return true;
case MIRType::Null:
case MIRType::Undefined:
*res = false;
return true;
case MIRType::Symbol:
*res = true;
return true;
case MIRType::BigInt:
*res = !toBigInt()->isZero();
return true;
case MIRType::String:
*res = toString()->length() != 0;
return true;
case MIRType::Object:
// TODO(Warp): Lazy groups have been removed.
// We have to call EmulatesUndefined but that reads obj->group->clasp
// and so it's racy when the object has a lazy group. The main callers
// of this (MTest, MNot) already know how to fold the object case, so
// just give up.
return false;
default:
MOZ_ASSERT(IsMagicType(type()));
return false;
}
}
HashNumber MWasmFloatConstant::valueHash() const {
#ifdef ENABLE_WASM_SIMD
return ConstantValueHash(type(), u.bits_[0] ^ u.bits_[1]);
#else
return ConstantValueHash(type(), u.bits_[0]);
#endif
}
bool MWasmFloatConstant::congruentTo(const MDefinition* ins) const {
return ins->isWasmFloatConstant() && type() == ins->type() &&
#ifdef ENABLE_WASM_SIMD
u.bits_[1] == ins->toWasmFloatConstant()->u.bits_[1] &&
#endif
u.bits_[0] == ins->toWasmFloatConstant()->u.bits_[0];
}
HashNumber MWasmNullConstant::valueHash() const {
return ConstantValueHash(MIRType::WasmAnyRef, 0);
}
#ifdef JS_JITSPEW
void MControlInstruction::printOpcode(GenericPrinter& out) const {
MDefinition::printOpcode(out);
for (size_t j = 0; j < numSuccessors(); j++) {
if (getSuccessor(j)) {
out.printf(" block%u", getSuccessor(j)->id());
} else {
out.printf(" (null-to-be-patched)");
}
}
}
void MCompare::printOpcode(GenericPrinter& out) const {
MDefinition::printOpcode(out);
out.printf(" %s", CodeName(jsop()));
}
void MTypeOfIs::printOpcode(GenericPrinter& out) const {
MDefinition::printOpcode(out);
out.printf(" %s", CodeName(jsop()));
const char* name = "";
switch (jstype()) {
case JSTYPE_UNDEFINED:
name = "undefined";
break;
case JSTYPE_OBJECT:
name = "object";
break;
case JSTYPE_FUNCTION:
name = "function";
break;
case JSTYPE_STRING:
name = "string";
break;
case JSTYPE_NUMBER:
name = "number";
break;
case JSTYPE_BOOLEAN:
name = "boolean";
break;
case JSTYPE_SYMBOL:
name = "symbol";
break;
case JSTYPE_BIGINT:
name = "bigint";
break;
# ifdef ENABLE_RECORD_TUPLE
case JSTYPE_RECORD:
case JSTYPE_TUPLE:
# endif
case JSTYPE_LIMIT:
MOZ_CRASH("Unexpected type");
}
out.printf(" '%s'", name);
}
void MLoadUnboxedScalar::printOpcode(GenericPrinter& out) const {
MDefinition::printOpcode(out);
out.printf(" %s", Scalar::name(storageType()));
}
void MLoadDataViewElement::printOpcode(GenericPrinter& out) const {
MDefinition::printOpcode(out);
out.printf(" %s", Scalar::name(storageType()));
}
void MAssertRange::printOpcode(GenericPrinter& out) const {
MDefinition::printOpcode(out);
out.put(" ");
assertedRange()->dump(out);
}
void MNearbyInt::printOpcode(GenericPrinter& out) const {
MDefinition::printOpcode(out);
const char* roundingModeStr = nullptr;
switch (roundingMode_) {
case RoundingMode::Up:
roundingModeStr = "(up)";
break;
case RoundingMode::Down:
roundingModeStr = "(down)";
break;
case RoundingMode::NearestTiesToEven:
roundingModeStr = "(nearest ties even)";
break;
case RoundingMode::TowardsZero:
roundingModeStr = "(towards zero)";
break;
}
out.printf(" %s", roundingModeStr);
}
#endif
AliasSet MRandom::getAliasSet() const { return AliasSet::Store(AliasSet::RNG); }
MDefinition* MSign::foldsTo(TempAllocator& alloc) {
MDefinition* input = getOperand(0);
if (!input->isConstant() ||
!input->toConstant()->isTypeRepresentableAsDouble()) {
return this;
}
double in = input->toConstant()->numberToDouble();
double out = js::math_sign_impl(in);
if (type() == MIRType::Int32) {
// Decline folding if this is an int32 operation, but the result type
// isn't an int32.
Value outValue = NumberValue(out);
if (!outValue.isInt32()) {
return this;
}
return MConstant::New(alloc, outValue);
}
return MConstant::New(alloc, DoubleValue(out));
}
const char* MMathFunction::FunctionName(UnaryMathFunction function) {
return GetUnaryMathFunctionName(function);
}
#ifdef JS_JITSPEW
void MMathFunction::printOpcode(GenericPrinter& out) const {
MDefinition::printOpcode(out);
out.printf(" %s", FunctionName(function()));
}
#endif
MDefinition* MMathFunction::foldsTo(TempAllocator& alloc) {
MDefinition* input = getOperand(0);
if (!input->isConstant() ||
!input->toConstant()->isTypeRepresentableAsDouble()) {
return this;
}
UnaryMathFunctionType funPtr = GetUnaryMathFunctionPtr(function());
double in = input->toConstant()->numberToDouble();
// The function pointer call can't GC.
JS::AutoSuppressGCAnalysis nogc;
double out = funPtr(in);
if (input->type() == MIRType::Float32) {
return MConstant::NewFloat32(alloc, out);
}
return MConstant::New(alloc, DoubleValue(out));
}
MDefinition* MAtomicIsLockFree::foldsTo(TempAllocator& alloc) {
MDefinition* input = getOperand(0);
if (!input->isConstant() || input->type() != MIRType::Int32) {
return this;
}
int32_t i = input->toConstant()->toInt32();
return MConstant::New(alloc, BooleanValue(AtomicOperations::isLockfreeJS(i)));
}
// Define |THIS_SLOT| as part of this translation unit, as it is used to
// specialized the parameterized |New| function calls introduced by
// TRIVIAL_NEW_WRAPPERS.
const int32_t MParameter::THIS_SLOT;
#ifdef JS_JITSPEW
void MParameter::printOpcode(GenericPrinter& out) const {
PrintOpcodeName(out, op());
if (index() == THIS_SLOT) {
out.printf(" THIS_SLOT");
} else {
out.printf(" %d", index());
}
}
#endif
HashNumber MParameter::valueHash() const {
HashNumber hash = MDefinition::valueHash();
hash = addU32ToHash(hash, index_);
return hash;
}
bool MParameter::congruentTo(const MDefinition* ins) const {
if (!ins->isParameter()) {
return false;