Source code

Revision control

Other Tools

1
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2
* vim: set ts=8 sts=2 et sw=2 tw=80:
3
* This Source Code Form is subject to the terms of the Mozilla Public
4
* License, v. 2.0. If a copy of the MPL was not distributed with this
5
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7
#include "jit/CodeGenerator.h"
8
9
#include "mozilla/Assertions.h"
10
#include "mozilla/Attributes.h"
11
#include "mozilla/Casting.h"
12
#include "mozilla/DebugOnly.h"
13
#include "mozilla/EnumeratedArray.h"
14
#include "mozilla/EnumeratedRange.h"
15
#include "mozilla/MathAlgorithms.h"
16
#include "mozilla/ScopeExit.h"
17
#include "mozilla/Unused.h"
18
19
#include <type_traits>
20
21
#include "jslibmath.h"
22
#include "jsmath.h"
23
#include "jsnum.h"
24
25
#include "builtin/Eval.h"
26
#include "builtin/MapObject.h"
27
#include "builtin/RegExp.h"
28
#include "builtin/SelfHostingDefines.h"
29
#include "builtin/String.h"
30
#include "builtin/TypedObject.h"
31
#include "gc/Nursery.h"
32
#include "irregexp/NativeRegExpMacroAssembler.h"
33
#include "jit/AtomicOperations.h"
34
#include "jit/BaselineCodeGen.h"
35
#include "jit/IonBuilder.h"
36
#include "jit/IonIC.h"
37
#include "jit/IonOptimizationLevels.h"
38
#include "jit/JitcodeMap.h"
39
#include "jit/JitSpewer.h"
40
#include "jit/Linker.h"
41
#include "jit/Lowering.h"
42
#include "jit/MIRGenerator.h"
43
#include "jit/MoveEmitter.h"
44
#include "jit/RangeAnalysis.h"
45
#include "jit/SharedICHelpers.h"
46
#include "jit/StackSlotAllocator.h"
47
#include "jit/VMFunctions.h"
48
#include "js/RegExpFlags.h" // JS::RegExpFlag
49
#include "util/CheckedArithmetic.h"
50
#include "util/Unicode.h"
51
#include "vm/AsyncFunction.h"
52
#include "vm/AsyncIteration.h"
53
#include "vm/EqualityOperations.h" // js::SameValue
54
#include "vm/MatchPairs.h"
55
#include "vm/RegExpObject.h"
56
#include "vm/RegExpStatics.h"
57
#include "vm/StringType.h"
58
#include "vm/TraceLogging.h"
59
#include "vm/TypedArrayObject.h"
60
#ifdef MOZ_VTUNE
61
# include "vtune/VTuneWrapper.h"
62
#endif
63
#include "wasm/WasmGC.h"
64
#include "wasm/WasmStubs.h"
65
66
#include "builtin/Boolean-inl.h"
67
#include "jit/MacroAssembler-inl.h"
68
#include "jit/shared/CodeGenerator-shared-inl.h"
69
#include "jit/shared/Lowering-shared-inl.h"
70
#include "jit/TemplateObject-inl.h"
71
#include "jit/VMFunctionList-inl.h"
72
#include "vm/Interpreter-inl.h"
73
#include "vm/JSScript-inl.h"
74
75
using namespace js;
76
using namespace js::jit;
77
78
using JS::GenericNaN;
79
using mozilla::AssertedCast;
80
using mozilla::DebugOnly;
81
using mozilla::FloatingPoint;
82
using mozilla::Maybe;
83
using mozilla::NegativeInfinity;
84
using mozilla::PositiveInfinity;
85
86
namespace js {
87
namespace jit {
88
89
#ifdef CHECK_OSIPOINT_REGISTERS
90
template <class Op>
91
static void HandleRegisterDump(Op op, MacroAssembler& masm,
92
LiveRegisterSet liveRegs, Register activation,
93
Register scratch) {
94
const size_t baseOffset = JitActivation::offsetOfRegs();
95
96
// Handle live GPRs.
97
for (GeneralRegisterIterator iter(liveRegs.gprs()); iter.more(); ++iter) {
98
Register reg = *iter;
99
Address dump(activation, baseOffset + RegisterDump::offsetOfRegister(reg));
100
101
if (reg == activation) {
102
// To use the original value of the activation register (that's
103
// now on top of the stack), we need the scratch register.
104
masm.push(scratch);
105
masm.loadPtr(Address(masm.getStackPointer(), sizeof(uintptr_t)), scratch);
106
op(scratch, dump);
107
masm.pop(scratch);
108
} else {
109
op(reg, dump);
110
}
111
}
112
113
// Handle live FPRs.
114
for (FloatRegisterIterator iter(liveRegs.fpus()); iter.more(); ++iter) {
115
FloatRegister reg = *iter;
116
Address dump(activation, baseOffset + RegisterDump::offsetOfRegister(reg));
117
op(reg, dump);
118
}
119
}
120
121
class StoreOp {
122
MacroAssembler& masm;
123
124
public:
125
explicit StoreOp(MacroAssembler& masm) : masm(masm) {}
126
127
void operator()(Register reg, Address dump) { masm.storePtr(reg, dump); }
128
void operator()(FloatRegister reg, Address dump) {
129
if (reg.isDouble()) {
130
masm.storeDouble(reg, dump);
131
} else if (reg.isSingle()) {
132
masm.storeFloat32(reg, dump);
133
# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
134
} else if (reg.isSimd128()) {
135
masm.storeUnalignedSimd128Float(reg, dump);
136
# endif
137
} else {
138
MOZ_CRASH("Unexpected register type.");
139
}
140
}
141
};
142
143
class VerifyOp {
144
MacroAssembler& masm;
145
Label* failure_;
146
147
public:
148
VerifyOp(MacroAssembler& masm, Label* failure)
149
: masm(masm), failure_(failure) {}
150
151
void operator()(Register reg, Address dump) {
152
masm.branchPtr(Assembler::NotEqual, dump, reg, failure_);
153
}
154
void operator()(FloatRegister reg, Address dump) {
155
if (reg.isDouble()) {
156
ScratchDoubleScope scratch(masm);
157
masm.loadDouble(dump, scratch);
158
masm.branchDouble(Assembler::DoubleNotEqual, scratch, reg, failure_);
159
} else if (reg.isSingle()) {
160
ScratchFloat32Scope scratch(masm);
161
masm.loadFloat32(dump, scratch);
162
masm.branchFloat(Assembler::DoubleNotEqual, scratch, reg, failure_);
163
}
164
165
// :TODO: (Bug 1133745) Add support to verify SIMD registers.
166
}
167
};
168
169
void CodeGenerator::verifyOsiPointRegs(LSafepoint* safepoint) {
170
// Ensure the live registers stored by callVM did not change between
171
// the call and this OsiPoint. Try-catch relies on this invariant.
172
173
// Load pointer to the JitActivation in a scratch register.
174
AllocatableGeneralRegisterSet allRegs(GeneralRegisterSet::All());
175
Register scratch = allRegs.takeAny();
176
masm.push(scratch);
177
masm.loadJitActivation(scratch);
178
179
// If we should not check registers (because the instruction did not call
180
// into the VM, or a GC happened), we're done.
181
Label failure, done;
182
Address checkRegs(scratch, JitActivation::offsetOfCheckRegs());
183
masm.branch32(Assembler::Equal, checkRegs, Imm32(0), &done);
184
185
// Having more than one VM function call made in one visit function at
186
// runtime is a sec-ciritcal error, because if we conservatively assume that
187
// one of the function call can re-enter Ion, then the invalidation process
188
// will potentially add a call at a random location, by patching the code
189
// before the return address.
190
masm.branch32(Assembler::NotEqual, checkRegs, Imm32(1), &failure);
191
192
// Set checkRegs to 0, so that we don't try to verify registers after we
193
// return from this script to the caller.
194
masm.store32(Imm32(0), checkRegs);
195
196
// Ignore clobbered registers. Some instructions (like LValueToInt32) modify
197
// temps after calling into the VM. This is fine because no other
198
// instructions (including this OsiPoint) will depend on them. Also
199
// backtracking can also use the same register for an input and an output.
200
// These are marked as clobbered and shouldn't get checked.
201
LiveRegisterSet liveRegs;
202
liveRegs.set() = RegisterSet::Intersect(
203
safepoint->liveRegs().set(),
204
RegisterSet::Not(safepoint->clobberedRegs().set()));
205
206
VerifyOp op(masm, &failure);
207
HandleRegisterDump<VerifyOp>(op, masm, liveRegs, scratch, allRegs.getAny());
208
209
masm.jump(&done);
210
211
// Do not profile the callWithABI that occurs below. This is to avoid a
212
// rare corner case that occurs when profiling interacts with itself:
213
//
214
// When slow profiling assertions are turned on, FunctionBoundary ops
215
// (which update the profiler pseudo-stack) may emit a callVM, which
216
// forces them to have an osi point associated with them. The
217
// FunctionBoundary for inline function entry is added to the caller's
218
// graph with a PC from the caller's code, but during codegen it modifies
219
// Gecko Profiler instrumentation to add the callee as the current top-most
220
// script. When codegen gets to the OSIPoint, and the callWithABI below is
221
// emitted, the codegen thinks that the current frame is the callee, but
222
// the PC it's using from the OSIPoint refers to the caller. This causes
223
// the profiler instrumentation of the callWithABI below to ASSERT, since
224
// the script and pc are mismatched. To avoid this, we simply omit
225
// instrumentation for these callWithABIs.
226
227
// Any live register captured by a safepoint (other than temp registers)
228
// must remain unchanged between the call and the OsiPoint instruction.
229
masm.bind(&failure);
230
masm.assumeUnreachable("Modified registers between VM call and OsiPoint");
231
232
masm.bind(&done);
233
masm.pop(scratch);
234
}
235
236
bool CodeGenerator::shouldVerifyOsiPointRegs(LSafepoint* safepoint) {
237
if (!checkOsiPointRegisters) {
238
return false;
239
}
240
241
if (safepoint->liveRegs().emptyGeneral() &&
242
safepoint->liveRegs().emptyFloat()) {
243
return false; // No registers to check.
244
}
245
246
return true;
247
}
248
249
void CodeGenerator::resetOsiPointRegs(LSafepoint* safepoint) {
250
if (!shouldVerifyOsiPointRegs(safepoint)) {
251
return;
252
}
253
254
// Set checkRegs to 0. If we perform a VM call, the instruction
255
// will set it to 1.
256
AllocatableGeneralRegisterSet allRegs(GeneralRegisterSet::All());
257
Register scratch = allRegs.takeAny();
258
masm.push(scratch);
259
masm.loadJitActivation(scratch);
260
Address checkRegs(scratch, JitActivation::offsetOfCheckRegs());
261
masm.store32(Imm32(0), checkRegs);
262
masm.pop(scratch);
263
}
264
265
static void StoreAllLiveRegs(MacroAssembler& masm, LiveRegisterSet liveRegs) {
266
// Store a copy of all live registers before performing the call.
267
// When we reach the OsiPoint, we can use this to check nothing
268
// modified them in the meantime.
269
270
// Load pointer to the JitActivation in a scratch register.
271
AllocatableGeneralRegisterSet allRegs(GeneralRegisterSet::All());
272
Register scratch = allRegs.takeAny();
273
masm.push(scratch);
274
masm.loadJitActivation(scratch);
275
276
Address checkRegs(scratch, JitActivation::offsetOfCheckRegs());
277
masm.add32(Imm32(1), checkRegs);
278
279
StoreOp op(masm);
280
HandleRegisterDump<StoreOp>(op, masm, liveRegs, scratch, allRegs.getAny());
281
282
masm.pop(scratch);
283
}
284
#endif // CHECK_OSIPOINT_REGISTERS
285
286
// Before doing any call to Cpp, you should ensure that volatile
287
// registers are evicted by the register allocator.
288
void CodeGenerator::callVMInternal(VMFunctionId id, LInstruction* ins,
289
const Register* dynStack) {
290
TrampolinePtr code = gen->jitRuntime()->getVMWrapper(id);
291
const VMFunctionData& fun = GetVMFunction(id);
292
293
#ifdef DEBUG
294
if (ins->mirRaw()) {
295
MOZ_ASSERT(ins->mirRaw()->isInstruction());
296
MInstruction* mir = ins->mirRaw()->toInstruction();
297
MOZ_ASSERT_IF(mir->needsResumePoint(), mir->resumePoint());
298
}
299
#endif
300
301
// Stack is:
302
// ... frame ...
303
// [args]
304
#ifdef DEBUG
305
MOZ_ASSERT(pushedArgs_ == fun.explicitArgs);
306
pushedArgs_ = 0;
307
#endif
308
309
#ifdef CHECK_OSIPOINT_REGISTERS
310
if (shouldVerifyOsiPointRegs(ins->safepoint())) {
311
StoreAllLiveRegs(masm, ins->safepoint()->liveRegs());
312
}
313
#endif
314
315
// Push an exit frame descriptor. If |dynStack| is a valid pointer to a
316
// register, then its value is added to the value of the |framePushed()| to
317
// fill the frame descriptor.
318
if (dynStack) {
319
masm.addPtr(Imm32(masm.framePushed()), *dynStack);
320
masm.makeFrameDescriptor(*dynStack, FrameType::IonJS,
321
ExitFrameLayout::Size());
322
masm.Push(*dynStack); // descriptor
323
} else {
324
masm.pushStaticFrameDescriptor(FrameType::IonJS, ExitFrameLayout::Size());
325
}
326
327
// Call the wrapper function. The wrapper is in charge to unwind the stack
328
// when returning from the call. Failures are handled with exceptions based
329
// on the return value of the C functions. To guard the outcome of the
330
// returned value, use another LIR instruction.
331
uint32_t callOffset = masm.callJit(code);
332
markSafepointAt(callOffset, ins);
333
334
// Remove rest of the frame left on the stack. We remove the return address
335
// which is implicitly poped when returning.
336
int framePop = sizeof(ExitFrameLayout) - sizeof(void*);
337
338
// Pop arguments from framePushed.
339
masm.implicitPop(fun.explicitStackSlots() * sizeof(void*) + framePop);
340
// Stack is:
341
// ... frame ...
342
}
343
344
template <typename Fn, Fn fn>
345
void CodeGenerator::callVM(LInstruction* ins, const Register* dynStack) {
346
VMFunctionId id = VMFunctionToId<Fn, fn>::id;
347
callVMInternal(id, ins, dynStack);
348
}
349
350
// ArgSeq store arguments for OutOfLineCallVM.
351
//
352
// OutOfLineCallVM are created with "oolCallVM" function. The third argument of
353
// this function is an instance of a class which provides a "generate" in charge
354
// of pushing the argument, with "pushArg", for a VMFunction.
355
//
356
// Such list of arguments can be created by using the "ArgList" function which
357
// creates one instance of "ArgSeq", where the type of the arguments are
358
// inferred from the type of the arguments.
359
//
360
// The list of arguments must be written in the same order as if you were
361
// calling the function in C++.
362
//
363
// Example:
364
// ArgList(ToRegister(lir->lhs()), ToRegister(lir->rhs()))
365
366
template <typename... ArgTypes>
367
class ArgSeq;
368
369
template <>
370
class ArgSeq<> {
371
public:
372
ArgSeq() {}
373
374
inline void generate(CodeGenerator* codegen) const {}
375
376
#ifdef DEBUG
377
static constexpr size_t numArgs = 0;
378
#endif
379
};
380
381
template <typename HeadType, typename... TailTypes>
382
class ArgSeq<HeadType, TailTypes...> : public ArgSeq<TailTypes...> {
383
private:
384
using RawHeadType = typename mozilla::RemoveReference<HeadType>::Type;
385
RawHeadType head_;
386
387
public:
388
template <typename ProvidedHead, typename... ProvidedTail>
389
explicit ArgSeq(ProvidedHead&& head, ProvidedTail&&... tail)
390
: ArgSeq<TailTypes...>(std::forward<ProvidedTail>(tail)...),
391
head_(std::forward<ProvidedHead>(head)) {}
392
393
// Arguments are pushed in reverse order, from last argument to first
394
// argument.
395
inline void generate(CodeGenerator* codegen) const {
396
this->ArgSeq<TailTypes...>::generate(codegen);
397
codegen->pushArg(head_);
398
}
399
400
#ifdef DEBUG
401
static constexpr size_t numArgs = sizeof...(TailTypes) + 1;
402
#endif
403
};
404
405
template <typename... ArgTypes>
406
inline ArgSeq<ArgTypes...> ArgList(ArgTypes&&... args) {
407
return ArgSeq<ArgTypes...>(std::forward<ArgTypes>(args)...);
408
}
409
410
// Store wrappers, to generate the right move of data after the VM call.
411
412
struct StoreNothing {
413
inline void generate(CodeGenerator* codegen) const {}
414
inline LiveRegisterSet clobbered() const {
415
return LiveRegisterSet(); // No register gets clobbered
416
}
417
};
418
419
class StoreRegisterTo {
420
private:
421
Register out_;
422
423
public:
424
explicit StoreRegisterTo(Register out) : out_(out) {}
425
426
inline void generate(CodeGenerator* codegen) const {
427
// It's okay to use storePointerResultTo here - the VMFunction wrapper
428
// ensures the upper bytes are zero for bool/int32 return values.
429
codegen->storePointerResultTo(out_);
430
}
431
inline LiveRegisterSet clobbered() const {
432
LiveRegisterSet set;
433
set.add(out_);
434
return set;
435
}
436
};
437
438
class StoreFloatRegisterTo {
439
private:
440
FloatRegister out_;
441
442
public:
443
explicit StoreFloatRegisterTo(FloatRegister out) : out_(out) {}
444
445
inline void generate(CodeGenerator* codegen) const {
446
codegen->storeFloatResultTo(out_);
447
}
448
inline LiveRegisterSet clobbered() const {
449
LiveRegisterSet set;
450
set.add(out_);
451
return set;
452
}
453
};
454
455
template <typename Output>
456
class StoreValueTo_ {
457
private:
458
Output out_;
459
460
public:
461
explicit StoreValueTo_(const Output& out) : out_(out) {}
462
463
inline void generate(CodeGenerator* codegen) const {
464
codegen->storeResultValueTo(out_);
465
}
466
inline LiveRegisterSet clobbered() const {
467
LiveRegisterSet set;
468
set.add(out_);
469
return set;
470
}
471
};
472
473
template <typename Output>
474
StoreValueTo_<Output> StoreValueTo(const Output& out) {
475
return StoreValueTo_<Output>(out);
476
}
477
478
template <typename Fn, Fn fn, class ArgSeq, class StoreOutputTo>
479
class OutOfLineCallVM : public OutOfLineCodeBase<CodeGenerator> {
480
private:
481
LInstruction* lir_;
482
ArgSeq args_;
483
StoreOutputTo out_;
484
485
public:
486
OutOfLineCallVM(LInstruction* lir, const ArgSeq& args,
487
const StoreOutputTo& out)
488
: lir_(lir), args_(args), out_(out) {}
489
490
void accept(CodeGenerator* codegen) override {
491
codegen->visitOutOfLineCallVM(this);
492
}
493
494
LInstruction* lir() const { return lir_; }
495
const ArgSeq& args() const { return args_; }
496
const StoreOutputTo& out() const { return out_; }
497
};
498
499
template <typename Fn, Fn fn, class ArgSeq, class StoreOutputTo>
500
OutOfLineCode* CodeGenerator::oolCallVM(LInstruction* lir, const ArgSeq& args,
501
const StoreOutputTo& out) {
502
MOZ_ASSERT(lir->mirRaw());
503
MOZ_ASSERT(lir->mirRaw()->isInstruction());
504
505
#ifdef DEBUG
506
VMFunctionId id = VMFunctionToId<Fn, fn>::id;
507
const VMFunctionData& fun = GetVMFunction(id);
508
MOZ_ASSERT(fun.explicitArgs == args.numArgs);
509
MOZ_ASSERT(fun.returnsData() !=
510
(mozilla::IsSame<StoreOutputTo, StoreNothing>::value));
511
#endif
512
513
OutOfLineCode* ool = new (alloc())
514
OutOfLineCallVM<Fn, fn, ArgSeq, StoreOutputTo>(lir, args, out);
515
addOutOfLineCode(ool, lir->mirRaw()->toInstruction());
516
return ool;
517
}
518
519
template <typename Fn, Fn fn, class ArgSeq, class StoreOutputTo>
520
void CodeGenerator::visitOutOfLineCallVM(
521
OutOfLineCallVM<Fn, fn, ArgSeq, StoreOutputTo>* ool) {
522
LInstruction* lir = ool->lir();
523
524
saveLive(lir);
525
ool->args().generate(this);
526
callVM<Fn, fn>(lir);
527
ool->out().generate(this);
528
restoreLiveIgnore(lir, ool->out().clobbered());
529
masm.jump(ool->rejoin());
530
}
531
532
class OutOfLineICFallback : public OutOfLineCodeBase<CodeGenerator> {
533
private:
534
LInstruction* lir_;
535
size_t cacheIndex_;
536
size_t cacheInfoIndex_;
537
538
public:
539
OutOfLineICFallback(LInstruction* lir, size_t cacheIndex,
540
size_t cacheInfoIndex)
541
: lir_(lir), cacheIndex_(cacheIndex), cacheInfoIndex_(cacheInfoIndex) {}
542
543
void bind(MacroAssembler* masm) override {
544
// The binding of the initial jump is done in
545
// CodeGenerator::visitOutOfLineICFallback.
546
}
547
548
size_t cacheIndex() const { return cacheIndex_; }
549
size_t cacheInfoIndex() const { return cacheInfoIndex_; }
550
LInstruction* lir() const { return lir_; }
551
552
void accept(CodeGenerator* codegen) override {
553
codegen->visitOutOfLineICFallback(this);
554
}
555
};
556
557
void CodeGeneratorShared::addIC(LInstruction* lir, size_t cacheIndex) {
558
if (cacheIndex == SIZE_MAX) {
559
masm.setOOM();
560
return;
561
}
562
563
DataPtr<IonIC> cache(this, cacheIndex);
564
MInstruction* mir = lir->mirRaw()->toInstruction();
565
if (mir->resumePoint()) {
566
cache->setScriptedLocation(mir->block()->info().script(),
567
mir->resumePoint()->pc());
568
} else {
569
cache->setIdempotent();
570
}
571
572
Register temp = cache->scratchRegisterForEntryJump();
573
icInfo_.back().icOffsetForJump = masm.movWithPatch(ImmWord(-1), temp);
574
masm.jump(Address(temp, 0));
575
576
MOZ_ASSERT(!icInfo_.empty());
577
578
OutOfLineICFallback* ool =
579
new (alloc()) OutOfLineICFallback(lir, cacheIndex, icInfo_.length() - 1);
580
addOutOfLineCode(ool, mir);
581
582
masm.bind(ool->rejoin());
583
cache->setRejoinOffset(CodeOffset(ool->rejoin()->offset()));
584
}
585
586
void CodeGenerator::visitOutOfLineICFallback(OutOfLineICFallback* ool) {
587
LInstruction* lir = ool->lir();
588
size_t cacheIndex = ool->cacheIndex();
589
size_t cacheInfoIndex = ool->cacheInfoIndex();
590
591
DataPtr<IonIC> ic(this, cacheIndex);
592
593
// Register the location of the OOL path in the IC.
594
ic->setFallbackOffset(CodeOffset(masm.currentOffset()));
595
596
switch (ic->kind()) {
597
case CacheKind::GetProp:
598
case CacheKind::GetElem: {
599
IonGetPropertyIC* getPropIC = ic->asGetPropertyIC();
600
601
saveLive(lir);
602
603
pushArg(getPropIC->id());
604
pushArg(getPropIC->value());
605
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
606
pushArg(ImmGCPtr(gen->info().script()));
607
608
using Fn = bool (*)(JSContext*, HandleScript, IonGetPropertyIC*,
609
HandleValue, HandleValue, MutableHandleValue);
610
callVM<Fn, IonGetPropertyIC::update>(lir);
611
612
StoreValueTo(getPropIC->output()).generate(this);
613
restoreLiveIgnore(lir, StoreValueTo(getPropIC->output()).clobbered());
614
615
masm.jump(ool->rejoin());
616
return;
617
}
618
case CacheKind::GetPropSuper:
619
case CacheKind::GetElemSuper: {
620
IonGetPropSuperIC* getPropSuperIC = ic->asGetPropSuperIC();
621
622
saveLive(lir);
623
624
pushArg(getPropSuperIC->id());
625
pushArg(getPropSuperIC->receiver());
626
pushArg(getPropSuperIC->object());
627
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
628
pushArg(ImmGCPtr(gen->info().script()));
629
630
using Fn =
631
bool (*)(JSContext*, HandleScript, IonGetPropSuperIC*, HandleObject,
632
HandleValue, HandleValue, MutableHandleValue);
633
callVM<Fn, IonGetPropSuperIC::update>(lir);
634
635
StoreValueTo(getPropSuperIC->output()).generate(this);
636
restoreLiveIgnore(lir,
637
StoreValueTo(getPropSuperIC->output()).clobbered());
638
639
masm.jump(ool->rejoin());
640
return;
641
}
642
case CacheKind::SetProp:
643
case CacheKind::SetElem: {
644
IonSetPropertyIC* setPropIC = ic->asSetPropertyIC();
645
646
saveLive(lir);
647
648
pushArg(setPropIC->rhs());
649
pushArg(setPropIC->id());
650
pushArg(setPropIC->object());
651
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
652
pushArg(ImmGCPtr(gen->info().script()));
653
654
using Fn = bool (*)(JSContext*, HandleScript, IonSetPropertyIC*,
655
HandleObject, HandleValue, HandleValue);
656
callVM<Fn, IonSetPropertyIC::update>(lir);
657
658
restoreLive(lir);
659
660
masm.jump(ool->rejoin());
661
return;
662
}
663
case CacheKind::GetName: {
664
IonGetNameIC* getNameIC = ic->asGetNameIC();
665
666
saveLive(lir);
667
668
pushArg(getNameIC->environment());
669
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
670
pushArg(ImmGCPtr(gen->info().script()));
671
672
using Fn = bool (*)(JSContext*, HandleScript, IonGetNameIC*, HandleObject,
673
MutableHandleValue);
674
callVM<Fn, IonGetNameIC::update>(lir);
675
676
StoreValueTo(getNameIC->output()).generate(this);
677
restoreLiveIgnore(lir, StoreValueTo(getNameIC->output()).clobbered());
678
679
masm.jump(ool->rejoin());
680
return;
681
}
682
case CacheKind::BindName: {
683
IonBindNameIC* bindNameIC = ic->asBindNameIC();
684
685
saveLive(lir);
686
687
pushArg(bindNameIC->environment());
688
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
689
pushArg(ImmGCPtr(gen->info().script()));
690
691
using Fn =
692
JSObject* (*)(JSContext*, HandleScript, IonBindNameIC*, HandleObject);
693
callVM<Fn, IonBindNameIC::update>(lir);
694
695
StoreRegisterTo(bindNameIC->output()).generate(this);
696
restoreLiveIgnore(lir, StoreRegisterTo(bindNameIC->output()).clobbered());
697
698
masm.jump(ool->rejoin());
699
return;
700
}
701
case CacheKind::GetIterator: {
702
IonGetIteratorIC* getIteratorIC = ic->asGetIteratorIC();
703
704
saveLive(lir);
705
706
pushArg(getIteratorIC->value());
707
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
708
pushArg(ImmGCPtr(gen->info().script()));
709
710
using Fn = JSObject* (*)(JSContext*, HandleScript, IonGetIteratorIC*,
711
HandleValue);
712
callVM<Fn, IonGetIteratorIC::update>(lir);
713
714
StoreRegisterTo(getIteratorIC->output()).generate(this);
715
restoreLiveIgnore(lir,
716
StoreRegisterTo(getIteratorIC->output()).clobbered());
717
718
masm.jump(ool->rejoin());
719
return;
720
}
721
case CacheKind::In: {
722
IonInIC* inIC = ic->asInIC();
723
724
saveLive(lir);
725
726
pushArg(inIC->object());
727
pushArg(inIC->key());
728
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
729
pushArg(ImmGCPtr(gen->info().script()));
730
731
using Fn = bool (*)(JSContext*, HandleScript, IonInIC*, HandleValue,
732
HandleObject, bool*);
733
callVM<Fn, IonInIC::update>(lir);
734
735
StoreRegisterTo(inIC->output()).generate(this);
736
restoreLiveIgnore(lir, StoreRegisterTo(inIC->output()).clobbered());
737
738
masm.jump(ool->rejoin());
739
return;
740
}
741
case CacheKind::HasOwn: {
742
IonHasOwnIC* hasOwnIC = ic->asHasOwnIC();
743
744
saveLive(lir);
745
746
pushArg(hasOwnIC->id());
747
pushArg(hasOwnIC->value());
748
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
749
pushArg(ImmGCPtr(gen->info().script()));
750
751
using Fn = bool (*)(JSContext*, HandleScript, IonHasOwnIC*, HandleValue,
752
HandleValue, int32_t*);
753
callVM<Fn, IonHasOwnIC::update>(lir);
754
755
StoreRegisterTo(hasOwnIC->output()).generate(this);
756
restoreLiveIgnore(lir, StoreRegisterTo(hasOwnIC->output()).clobbered());
757
758
masm.jump(ool->rejoin());
759
return;
760
}
761
case CacheKind::InstanceOf: {
762
IonInstanceOfIC* hasInstanceOfIC = ic->asInstanceOfIC();
763
764
saveLive(lir);
765
766
pushArg(hasInstanceOfIC->rhs());
767
pushArg(hasInstanceOfIC->lhs());
768
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
769
pushArg(ImmGCPtr(gen->info().script()));
770
771
using Fn = bool (*)(JSContext*, HandleScript, IonInstanceOfIC*,
772
HandleValue lhs, HandleObject rhs, bool* res);
773
callVM<Fn, IonInstanceOfIC::update>(lir);
774
775
StoreRegisterTo(hasInstanceOfIC->output()).generate(this);
776
restoreLiveIgnore(lir,
777
StoreRegisterTo(hasInstanceOfIC->output()).clobbered());
778
779
masm.jump(ool->rejoin());
780
return;
781
}
782
case CacheKind::UnaryArith: {
783
IonUnaryArithIC* unaryArithIC = ic->asUnaryArithIC();
784
785
saveLive(lir);
786
787
pushArg(unaryArithIC->input());
788
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
789
pushArg(ImmGCPtr(gen->info().script()));
790
791
using Fn = bool (*)(JSContext * cx, HandleScript outerScript,
792
IonUnaryArithIC * stub, HandleValue val,
793
MutableHandleValue res);
794
callVM<Fn, IonUnaryArithIC::update>(lir);
795
796
StoreValueTo(unaryArithIC->output()).generate(this);
797
restoreLiveIgnore(lir, StoreValueTo(unaryArithIC->output()).clobbered());
798
799
masm.jump(ool->rejoin());
800
return;
801
}
802
case CacheKind::BinaryArith: {
803
IonBinaryArithIC* binaryArithIC = ic->asBinaryArithIC();
804
805
saveLive(lir);
806
807
pushArg(binaryArithIC->rhs());
808
pushArg(binaryArithIC->lhs());
809
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
810
pushArg(ImmGCPtr(gen->info().script()));
811
812
using Fn = bool (*)(JSContext * cx, HandleScript outerScript,
813
IonBinaryArithIC * stub, HandleValue lhs,
814
HandleValue rhs, MutableHandleValue res);
815
callVM<Fn, IonBinaryArithIC::update>(lir);
816
817
StoreValueTo(binaryArithIC->output()).generate(this);
818
restoreLiveIgnore(lir, StoreValueTo(binaryArithIC->output()).clobbered());
819
820
masm.jump(ool->rejoin());
821
return;
822
}
823
case CacheKind::Compare: {
824
IonCompareIC* compareIC = ic->asCompareIC();
825
826
saveLive(lir);
827
828
pushArg(compareIC->rhs());
829
pushArg(compareIC->lhs());
830
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
831
pushArg(ImmGCPtr(gen->info().script()));
832
833
using Fn = bool (*)(JSContext * cx, HandleScript outerScript,
834
IonCompareIC * stub, HandleValue lhs, HandleValue rhs,
835
bool* res);
836
callVM<Fn, IonCompareIC::update>(lir);
837
838
StoreRegisterTo(compareIC->output()).generate(this);
839
restoreLiveIgnore(lir, StoreRegisterTo(compareIC->output()).clobbered());
840
841
masm.jump(ool->rejoin());
842
return;
843
}
844
case CacheKind::Call:
845
case CacheKind::TypeOf:
846
case CacheKind::ToBool:
847
case CacheKind::GetIntrinsic:
848
case CacheKind::NewObject:
849
MOZ_CRASH("Unsupported IC");
850
}
851
MOZ_CRASH();
852
}
853
854
StringObject* MNewStringObject::templateObj() const {
855
return &templateObj_->as<StringObject>();
856
}
857
858
CodeGenerator::CodeGenerator(MIRGenerator* gen, LIRGraph* graph,
859
MacroAssembler* masm)
860
: CodeGeneratorSpecific(gen, graph, masm),
861
ionScriptLabels_(gen->alloc()),
862
scriptCounts_(nullptr),
863
realmStubsToReadBarrier_(0) {}
864
865
CodeGenerator::~CodeGenerator() { js_delete(scriptCounts_); }
866
867
class OutOfLineZeroIfNaN : public OutOfLineCodeBase<CodeGenerator> {
868
LInstruction* lir_;
869
FloatRegister input_;
870
Register output_;
871
872
public:
873
OutOfLineZeroIfNaN(LInstruction* lir, FloatRegister input, Register output)
874
: lir_(lir), input_(input), output_(output) {}
875
876
void accept(CodeGenerator* codegen) override {
877
codegen->visitOutOfLineZeroIfNaN(this);
878
}
879
LInstruction* lir() const { return lir_; }
880
FloatRegister input() const { return input_; }
881
Register output() const { return output_; }
882
};
883
884
void CodeGenerator::visitValueToInt32(LValueToInt32* lir) {
885
ValueOperand operand = ToValue(lir, LValueToInt32::Input);
886
Register output = ToRegister(lir->output());
887
FloatRegister temp = ToFloatRegister(lir->tempFloat());
888
889
MDefinition* input;
890
if (lir->mode() == LValueToInt32::NORMAL) {
891
input = lir->mirNormal()->input();
892
} else if (lir->mode() == LValueToInt32::TRUNCATE_NOWRAP) {
893
input = lir->mirTruncateNoWrap()->input();
894
} else {
895
input = lir->mirTruncate()->input();
896
}
897
898
Label fails;
899
if (lir->mode() == LValueToInt32::TRUNCATE) {
900
OutOfLineCode* oolDouble = oolTruncateDouble(temp, output, lir->mir());
901
902
// We can only handle strings in truncation contexts, like bitwise
903
// operations.
904
Label* stringEntry;
905
Label* stringRejoin;
906
Register stringReg;
907
if (input->mightBeType(MIRType::String)) {
908
stringReg = ToRegister(lir->temp());
909
using Fn = bool (*)(JSContext*, JSString*, double*);
910
OutOfLineCode* oolString = oolCallVM<Fn, StringToNumber>(
911
lir, ArgList(stringReg), StoreFloatRegisterTo(temp));
912
stringEntry = oolString->entry();
913
stringRejoin = oolString->rejoin();
914
} else {
915
stringReg = InvalidReg;
916
stringEntry = nullptr;
917
stringRejoin = nullptr;
918
}
919
920
masm.truncateValueToInt32(operand, input, stringEntry, stringRejoin,
921
oolDouble->entry(), stringReg, temp, output,
922
&fails);
923
masm.bind(oolDouble->rejoin());
924
} else if (lir->mode() == LValueToInt32::TRUNCATE_NOWRAP) {
925
auto* ool = new (alloc()) OutOfLineZeroIfNaN(lir, temp, output);
926
addOutOfLineCode(ool, lir->mir());
927
928
masm.truncateNoWrapValueToInt32(operand, input, temp, output, ool->entry(),
929
&fails);
930
masm.bind(ool->rejoin());
931
} else {
932
masm.convertValueToInt32(operand, input, temp, output, &fails,
933
lir->mirNormal()->canBeNegativeZero(),
934
lir->mirNormal()->conversion());
935
}
936
937
bailoutFrom(&fails, lir->snapshot());
938
}
939
940
void CodeGenerator::visitOutOfLineZeroIfNaN(OutOfLineZeroIfNaN* ool) {
941
FloatRegister input = ool->input();
942
Register output = ool->output();
943
944
// NaN triggers the failure path for branchTruncateDoubleToInt32() on x86,
945
// x64, and ARM64, so handle it here. In all other cases bail out.
946
947
Label fails;
948
if (input.isSingle()) {
949
masm.branchFloat(Assembler::DoubleOrdered, input, input, &fails);
950
} else {
951
masm.branchDouble(Assembler::DoubleOrdered, input, input, &fails);
952
}
953
954
// ToInteger(NaN) is 0.
955
masm.move32(Imm32(0), output);
956
masm.jump(ool->rejoin());
957
958
bailoutFrom(&fails, ool->lir()->snapshot());
959
}
960
961
void CodeGenerator::visitValueToDouble(LValueToDouble* lir) {
962
MToDouble* mir = lir->mir();
963
ValueOperand operand = ToValue(lir, LValueToDouble::Input);
964
FloatRegister output = ToFloatRegister(lir->output());
965
966
Label isDouble, isInt32, isBool, isNull, isUndefined, done;
967
bool hasBoolean = false, hasNull = false, hasUndefined = false;
968
969
{
970
ScratchTagScope tag(masm, operand);
971
masm.splitTagForTest(operand, tag);
972
973
masm.branchTestDouble(Assembler::Equal, tag, &isDouble);
974
masm.branchTestInt32(Assembler::Equal, tag, &isInt32);
975
976
if (mir->conversion() != MToFPInstruction::NumbersOnly) {
977
masm.branchTestBoolean(Assembler::Equal, tag, &isBool);
978
masm.branchTestUndefined(Assembler::Equal, tag, &isUndefined);
979
hasBoolean = true;
980
hasUndefined = true;
981
if (mir->conversion() != MToFPInstruction::NonNullNonStringPrimitives) {
982
masm.branchTestNull(Assembler::Equal, tag, &isNull);
983
hasNull = true;
984
}
985
}
986
}
987
988
bailout(lir->snapshot());
989
990
if (hasNull) {
991
masm.bind(&isNull);
992
masm.loadConstantDouble(0.0, output);
993
masm.jump(&done);
994
}
995
996
if (hasUndefined) {
997
masm.bind(&isUndefined);
998
masm.loadConstantDouble(GenericNaN(), output);
999
masm.jump(&done);
1000
}
1001
1002
if (hasBoolean) {
1003
masm.bind(&isBool);
1004
masm.boolValueToDouble(operand, output);
1005
masm.jump(&done);
1006
}
1007
1008
masm.bind(&isInt32);
1009
masm.int32ValueToDouble(operand, output);
1010
masm.jump(&done);
1011
1012
masm.bind(&isDouble);
1013
masm.unboxDouble(operand, output);
1014
masm.bind(&done);
1015
}
1016
1017
void CodeGenerator::visitValueToFloat32(LValueToFloat32* lir) {
1018
MToFloat32* mir = lir->mir();
1019
ValueOperand operand = ToValue(lir, LValueToFloat32::Input);
1020
FloatRegister output = ToFloatRegister(lir->output());
1021
1022
Label isDouble, isInt32, isBool, isNull, isUndefined, done;
1023
bool hasBoolean = false, hasNull = false, hasUndefined = false;
1024
1025
{
1026
ScratchTagScope tag(masm, operand);
1027
masm.splitTagForTest(operand, tag);
1028
1029
masm.branchTestDouble(Assembler::Equal, tag, &isDouble);
1030
masm.branchTestInt32(Assembler::Equal, tag, &isInt32);
1031
1032
if (mir->conversion() != MToFPInstruction::NumbersOnly) {
1033
masm.branchTestBoolean(Assembler::Equal, tag, &isBool);
1034
masm.branchTestUndefined(Assembler::Equal, tag, &isUndefined);
1035
hasBoolean = true;
1036
hasUndefined = true;
1037
if (mir->conversion() != MToFPInstruction::NonNullNonStringPrimitives) {
1038
masm.branchTestNull(Assembler::Equal, tag, &isNull);
1039
hasNull = true;
1040
}
1041
}
1042
}
1043
1044
bailout(lir->snapshot());
1045
1046
if (hasNull) {
1047
masm.bind(&isNull);
1048
masm.loadConstantFloat32(0.0f, output);
1049
masm.jump(&done);
1050
}
1051
1052
if (hasUndefined) {
1053
masm.bind(&isUndefined);
1054
masm.loadConstantFloat32(float(GenericNaN()), output);
1055
masm.jump(&done);
1056
}
1057
1058
if (hasBoolean) {
1059
masm.bind(&isBool);
1060
masm.boolValueToFloat32(operand, output);
1061
masm.jump(&done);
1062
}
1063
1064
masm.bind(&isInt32);
1065
masm.int32ValueToFloat32(operand, output);
1066
masm.jump(&done);
1067
1068
masm.bind(&isDouble);
1069
// ARM and MIPS may not have a double register available if we've
1070
// allocated output as a float32.
1071
#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS32)
1072
ScratchDoubleScope fpscratch(masm);
1073
masm.unboxDouble(operand, fpscratch);
1074
masm.convertDoubleToFloat32(fpscratch, output);
1075
#else
1076
masm.unboxDouble(operand, output);
1077
masm.convertDoubleToFloat32(output, output);
1078
#endif
1079
masm.bind(&done);
1080
}
1081
1082
void CodeGenerator::visitInt32ToDouble(LInt32ToDouble* lir) {
1083
masm.convertInt32ToDouble(ToRegister(lir->input()),
1084
ToFloatRegister(lir->output()));
1085
}
1086
1087
void CodeGenerator::visitFloat32ToDouble(LFloat32ToDouble* lir) {
1088
masm.convertFloat32ToDouble(ToFloatRegister(lir->input()),
1089
ToFloatRegister(lir->output()));
1090
}
1091
1092
void CodeGenerator::visitDoubleToFloat32(LDoubleToFloat32* lir) {
1093
masm.convertDoubleToFloat32(ToFloatRegister(lir->input()),
1094
ToFloatRegister(lir->output()));
1095
}
1096
1097
void CodeGenerator::visitInt32ToFloat32(LInt32ToFloat32* lir) {
1098
masm.convertInt32ToFloat32(ToRegister(lir->input()),
1099
ToFloatRegister(lir->output()));
1100
}
1101
1102
void CodeGenerator::visitDoubleToInt32(LDoubleToInt32* lir) {
1103
Label fail;
1104
FloatRegister input = ToFloatRegister(lir->input());
1105
Register output = ToRegister(lir->output());
1106
masm.convertDoubleToInt32(input, output, &fail,
1107
lir->mir()->canBeNegativeZero());
1108
bailoutFrom(&fail, lir->snapshot());
1109
}
1110
1111
void CodeGenerator::visitFloat32ToInt32(LFloat32ToInt32* lir) {
1112
Label fail;
1113
FloatRegister input = ToFloatRegister(lir->input());
1114
Register output = ToRegister(lir->output());
1115
masm.convertFloat32ToInt32(input, output, &fail,
1116
lir->mir()->canBeNegativeZero());
1117
bailoutFrom(&fail, lir->snapshot());
1118
}
1119
1120
void CodeGenerator::visitDoubleToIntegerInt32(LDoubleToIntegerInt32* lir) {
1121
FloatRegister input = ToFloatRegister(lir->input());
1122
Register output = ToRegister(lir->output());
1123
1124
auto* ool = new (alloc()) OutOfLineZeroIfNaN(lir, input, output);
1125
addOutOfLineCode(ool, lir->mir());
1126
1127
masm.branchTruncateDoubleToInt32(input, output, ool->entry());
1128
masm.bind(ool->rejoin());
1129
}
1130
1131
void CodeGenerator::visitFloat32ToIntegerInt32(LFloat32ToIntegerInt32* lir) {
1132
FloatRegister input = ToFloatRegister(lir->input());
1133
Register output = ToRegister(lir->output());
1134
1135
auto* ool = new (alloc()) OutOfLineZeroIfNaN(lir, input, output);
1136
addOutOfLineCode(ool, lir->mir());
1137
1138
masm.branchTruncateFloat32ToInt32(input, output, ool->entry());
1139
masm.bind(ool->rejoin());
1140
}
1141
1142
void CodeGenerator::emitOOLTestObject(Register objreg,
1143
Label* ifEmulatesUndefined,
1144
Label* ifDoesntEmulateUndefined,
1145
Register scratch) {
1146
saveVolatile(scratch);
1147
masm.setupUnalignedABICall(scratch);
1148
masm.passABIArg(objreg);
1149
masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, js::EmulatesUndefined));
1150
masm.storeCallBoolResult(scratch);
1151
restoreVolatile(scratch);
1152
1153
masm.branchIfTrueBool(scratch, ifEmulatesUndefined);
1154
masm.jump(ifDoesntEmulateUndefined);
1155
}
1156
1157
// Base out-of-line code generator for all tests of the truthiness of an
1158
// object, where the object might not be truthy. (Recall that per spec all
1159
// objects are truthy, but we implement the JSCLASS_EMULATES_UNDEFINED class
1160
// flag to permit objects to look like |undefined| in certain contexts,
1161
// including in object truthiness testing.) We check truthiness inline except
1162
// when we're testing it on a proxy (or if TI guarantees us that the specified
1163
// object will never emulate |undefined|), in which case out-of-line code will
1164
// call EmulatesUndefined for a conclusive answer.
1165
class OutOfLineTestObject : public OutOfLineCodeBase<CodeGenerator> {
1166
Register objreg_;
1167
Register scratch_;
1168
1169
Label* ifEmulatesUndefined_;
1170
Label* ifDoesntEmulateUndefined_;
1171
1172
#ifdef DEBUG
1173
bool initialized() { return ifEmulatesUndefined_ != nullptr; }
1174
#endif
1175
1176
public:
1177
OutOfLineTestObject()
1178
: ifEmulatesUndefined_(nullptr), ifDoesntEmulateUndefined_(nullptr) {}
1179
1180
void accept(CodeGenerator* codegen) final {
1181
MOZ_ASSERT(initialized());
1182
codegen->emitOOLTestObject(objreg_, ifEmulatesUndefined_,
1183
ifDoesntEmulateUndefined_, scratch_);
1184
}
1185
1186
// Specify the register where the object to be tested is found, labels to
1187
// jump to if the object is truthy or falsy, and a scratch register for
1188
// use in the out-of-line path.
1189
void setInputAndTargets(Register objreg, Label* ifEmulatesUndefined,
1190
Label* ifDoesntEmulateUndefined, Register scratch) {
1191
MOZ_ASSERT(!initialized());
1192
MOZ_ASSERT(ifEmulatesUndefined);
1193
objreg_ = objreg;
1194
scratch_ = scratch;
1195
ifEmulatesUndefined_ = ifEmulatesUndefined;
1196
ifDoesntEmulateUndefined_ = ifDoesntEmulateUndefined;
1197
}
1198
};
1199
1200
// A subclass of OutOfLineTestObject containing two extra labels, for use when
1201
// the ifTruthy/ifFalsy labels are needed in inline code as well as out-of-line
1202
// code. The user should bind these labels in inline code, and specify them as
1203
// targets via setInputAndTargets, as appropriate.
1204
class OutOfLineTestObjectWithLabels : public OutOfLineTestObject {
1205
Label label1_;
1206
Label label2_;
1207
1208
public:
1209
OutOfLineTestObjectWithLabels() {}
1210
1211
Label* label1() { return &label1_; }
1212
Label* label2() { return &label2_; }
1213
};
1214
1215
void CodeGenerator::testObjectEmulatesUndefinedKernel(
1216
Register objreg, Label* ifEmulatesUndefined,
1217
Label* ifDoesntEmulateUndefined, Register scratch,
1218
OutOfLineTestObject* ool) {
1219
ool->setInputAndTargets(objreg, ifEmulatesUndefined, ifDoesntEmulateUndefined,
1220
scratch);
1221
1222
// Perform a fast-path check of the object's class flags if the object's
1223
// not a proxy. Let out-of-line code handle the slow cases that require
1224
// saving registers, making a function call, and restoring registers.
1225
masm.branchIfObjectEmulatesUndefined(objreg, scratch, ool->entry(),
1226
ifEmulatesUndefined);
1227
}
1228
1229
void CodeGenerator::branchTestObjectEmulatesUndefined(
1230
Register objreg, Label* ifEmulatesUndefined,
1231
Label* ifDoesntEmulateUndefined, Register scratch,
1232
OutOfLineTestObject* ool) {
1233
MOZ_ASSERT(!ifDoesntEmulateUndefined->bound(),
1234
"ifDoesntEmulateUndefined will be bound to the fallthrough path");
1235
1236
testObjectEmulatesUndefinedKernel(objreg, ifEmulatesUndefined,
1237
ifDoesntEmulateUndefined, scratch, ool);
1238
masm.bind(ifDoesntEmulateUndefined);
1239
}
1240
1241
void CodeGenerator::testObjectEmulatesUndefined(Register objreg,
1242
Label* ifEmulatesUndefined,
1243
Label* ifDoesntEmulateUndefined,
1244
Register scratch,
1245
OutOfLineTestObject* ool) {
1246
testObjectEmulatesUndefinedKernel(objreg, ifEmulatesUndefined,
1247
ifDoesntEmulateUndefined, scratch, ool);
1248
masm.jump(ifDoesntEmulateUndefined);
1249
}
1250
1251
void CodeGenerator::testValueTruthyKernel(
1252
const ValueOperand& value, const LDefinition* scratch1,
1253
const LDefinition* scratch2, FloatRegister fr, Label* ifTruthy,
1254
Label* ifFalsy, OutOfLineTestObject* ool, MDefinition* valueMIR) {
1255
// Count the number of possible type tags we might have, so we'll know when
1256
// we've checked them all and hence can avoid emitting a tag check for the
1257
// last one. In particular, whenever tagCount is 1 that means we've tried
1258
// all but one of them already so we know exactly what's left based on the
1259
// mightBe* booleans.
1260
bool mightBeUndefined = valueMIR->mightBeType(MIRType::Undefined);
1261
bool mightBeNull = valueMIR->mightBeType(MIRType::Null);
1262
bool mightBeBoolean = valueMIR->mightBeType(MIRType::Boolean);
1263
bool mightBeInt32 = valueMIR->mightBeType(MIRType::Int32);
1264
bool mightBeObject = valueMIR->mightBeType(MIRType::Object);
1265
bool mightBeString = valueMIR->mightBeType(MIRType::String);
1266
bool mightBeSymbol = valueMIR->mightBeType(MIRType::Symbol);
1267
bool mightBeDouble = valueMIR->mightBeType(MIRType::Double);
1268
bool mightBeBigInt = valueMIR->mightBeType(MIRType::BigInt);
1269
int tagCount = int(mightBeUndefined) + int(mightBeNull) +
1270
int(mightBeBoolean) + int(mightBeInt32) + int(mightBeObject) +
1271
int(mightBeString) + int(mightBeSymbol) + int(mightBeDouble) +
1272
int(mightBeBigInt);
1273
1274
MOZ_ASSERT_IF(!valueMIR->emptyResultTypeSet(), tagCount > 0);
1275
1276
// If we know we're null or undefined, we're definitely falsy, no
1277
// need to even check the tag.
1278
if (int(mightBeNull) + int(mightBeUndefined) == tagCount) {
1279
masm.jump(ifFalsy);
1280
return;
1281
}
1282
1283
ScratchTagScope tag(masm, value);
1284
masm.splitTagForTest(value, tag);
1285
1286
if (mightBeUndefined) {
1287
MOZ_ASSERT(tagCount > 1);
1288
masm.branchTestUndefined(Assembler::Equal, tag, ifFalsy);
1289
--tagCount;
1290
}
1291
1292
if (mightBeNull) {
1293
MOZ_ASSERT(tagCount > 1);
1294
masm.branchTestNull(Assembler::Equal, tag, ifFalsy);
1295
--tagCount;
1296
}
1297
1298
if (mightBeBoolean) {
1299
MOZ_ASSERT(tagCount != 0);
1300
Label notBoolean;
1301
if (tagCount != 1) {
1302
masm.branchTestBoolean(Assembler::NotEqual, tag, &notBoolean);
1303
}
1304
{
1305
ScratchTagScopeRelease _(&tag);
1306
masm.branchTestBooleanTruthy(false, value, ifFalsy);
1307
}
1308
if (tagCount != 1) {
1309
masm.jump(ifTruthy);
1310
}
1311
// Else just fall through to truthiness.
1312
masm.bind(&notBoolean);
1313
--tagCount;
1314
}
1315
1316
if (mightBeInt32) {
1317
MOZ_ASSERT(tagCount != 0);
1318
Label notInt32;
1319
if (tagCount != 1) {
1320
masm.branchTestInt32(Assembler::NotEqual, tag, &notInt32);
1321
}
1322
{
1323
ScratchTagScopeRelease _(&tag);
1324
masm.branchTestInt32Truthy(false, value, ifFalsy);
1325
}
1326
if (tagCount != 1) {
1327
masm.jump(ifTruthy);
1328
}
1329
// Else just fall through to truthiness.
1330
masm.bind(&notInt32);
1331
--tagCount;
1332
}
1333
1334
if (mightBeObject) {
1335
MOZ_ASSERT(tagCount != 0);
1336
if (ool) {
1337
Label notObject;
1338
1339
if (tagCount != 1) {
1340
masm.branchTestObject(Assembler::NotEqual, tag, &notObject);
1341
}
1342
1343
{
1344
ScratchTagScopeRelease _(&tag);
1345
Register objreg = masm.extractObject(value, ToRegister(scratch1));
1346
testObjectEmulatesUndefined(objreg, ifFalsy, ifTruthy,
1347
ToRegister(scratch2), ool);
1348
}
1349
1350
masm.bind(&notObject);
1351
} else {
1352
if (tagCount != 1) {
1353
masm.branchTestObject(Assembler::Equal, tag, ifTruthy);
1354
}
1355
// Else just fall through to truthiness.
1356
}
1357
--tagCount;
1358
} else {
1359
MOZ_ASSERT(!ool,
1360
"We better not have an unused OOL path, since the code "
1361
"generator will try to "
1362
"generate code for it but we never set up its labels, which "
1363
"will cause null "
1364
"derefs of those labels.");
1365
}
1366
1367
if (mightBeString) {
1368
// Test if a string is non-empty.
1369
MOZ_ASSERT(tagCount != 0);
1370
Label notString;
1371
if (tagCount != 1) {
1372
masm.branchTestString(Assembler::NotEqual, tag, &notString);
1373
}
1374
{
1375
ScratchTagScopeRelease _(&tag);
1376
masm.branchTestStringTruthy(false, value, ifFalsy);
1377
}
1378
if (tagCount != 1) {
1379
masm.jump(ifTruthy);
1380
}
1381
// Else just fall through to truthiness.
1382
masm.bind(&notString);
1383
--tagCount;
1384
}
1385
1386
if (mightBeBigInt) {
1387
MOZ_ASSERT(tagCount != 0);
1388
Label notBigInt;
1389
if (tagCount != 1) {
1390
masm.branchTestBigInt(Assembler::NotEqual, tag, &notBigInt);
1391
}
1392
{
1393
ScratchTagScopeRelease _(&tag);
1394
masm.branchTestBigIntTruthy(false, value, ifFalsy);
1395
}
1396
if (tagCount != 1) {
1397
masm.jump(ifTruthy);
1398
}
1399
masm.bind(&notBigInt);
1400
--tagCount;
1401
}
1402
1403
if (mightBeSymbol) {
1404
// All symbols are truthy.
1405
MOZ_ASSERT(tagCount != 0);
1406
if (tagCount != 1) {
1407
masm.branchTestSymbol(Assembler::Equal, tag, ifTruthy);
1408
}
1409
// Else fall through to ifTruthy.
1410
--tagCount;
1411
}
1412
1413
if (mightBeDouble) {
1414
MOZ_ASSERT(tagCount == 1);
1415
// If we reach here the value is a double.
1416
{
1417
ScratchTagScopeRelease _(&tag);
1418
masm.unboxDouble(value, fr);
1419
masm.branchTestDoubleTruthy(false, fr, ifFalsy);
1420
}
1421
--tagCount;
1422
}
1423
1424
MOZ_ASSERT(tagCount == 0);
1425
1426
// Fall through for truthy.
1427
}
1428
1429
void CodeGenerator::testValueTruthy(const ValueOperand& value,
1430
const LDefinition* scratch1,
1431
const LDefinition* scratch2,
1432
FloatRegister fr, Label* ifTruthy,
1433
Label* ifFalsy, OutOfLineTestObject* ool,
1434
MDefinition* valueMIR) {
1435
testValueTruthyKernel(value, scratch1, scratch2, fr, ifTruthy, ifFalsy, ool,
1436
valueMIR);
1437
masm.jump(ifTruthy);
1438
}
1439
1440
void CodeGenerator::visitTestOAndBranch(LTestOAndBranch* lir) {
1441
MIRType inputType = lir->mir()->input()->type();
1442
MOZ_ASSERT(inputType == MIRType::ObjectOrNull ||
1443
lir->mir()->operandMightEmulateUndefined(),
1444
"If the object couldn't emulate undefined, this should have been "
1445
"folded.");
1446
1447
Label* truthy = getJumpLabelForBranch(lir->ifTruthy());
1448
Label* falsy = getJumpLabelForBranch(lir->ifFalsy());
1449
Register input = ToRegister(lir->input());
1450
1451
if (lir->mir()->operandMightEmulateUndefined()) {
1452
if (inputType == MIRType::ObjectOrNull) {
1453
masm.branchTestPtr(Assembler::Zero, input, input, falsy);
1454
}
1455
1456
OutOfLineTestObject* ool = new (alloc()) OutOfLineTestObject();
1457
addOutOfLineCode(ool, lir->mir());
1458
1459
testObjectEmulatesUndefined(input, falsy, truthy, ToRegister(lir->temp()),
1460
ool);
1461
} else {
1462
MOZ_ASSERT(inputType == MIRType::ObjectOrNull);
1463
testZeroEmitBranch(Assembler::NotEqual, input, lir->ifTruthy(),
1464
lir->ifFalsy());
1465
}
1466
}
1467
1468
void CodeGenerator::visitTestVAndBranch(LTestVAndBranch* lir) {
1469
OutOfLineTestObject* ool = nullptr;
1470
MDefinition* input = lir->mir()->input();
1471
// Unfortunately, it's possible that someone (e.g. phi elimination) switched
1472
// out our input after we did cacheOperandMightEmulateUndefined. So we
1473
// might think it can emulate undefined _and_ know that it can't be an
1474
// object.
1475
if (lir->mir()->operandMightEmulateUndefined() &&
1476
input->mightBeType(MIRType::Object)) {
1477
ool = new (alloc()) OutOfLineTestObject();
1478
addOutOfLineCode(ool, lir->mir());
1479
}
1480
1481
Label* truthy = getJumpLabelForBranch(lir->ifTruthy());
1482
Label* falsy = getJumpLabelForBranch(lir->ifFalsy());
1483
1484
testValueTruthy(ToValue(lir, LTestVAndBranch::Input), lir->temp1(),
1485
lir->temp2(), ToFloatRegister(lir->tempFloat()), truthy,
1486
falsy, ool, input);
1487
}
1488
1489
void CodeGenerator::visitFunctionDispatch(LFunctionDispatch* lir) {
1490
MFunctionDispatch* mir = lir->mir();
1491
Register input = ToRegister(lir->input());
1492
1493
// Compare function pointers
1494
for (size_t i = 0; i < mir->numCases(); i++) {
1495
MOZ_ASSERT(i < mir->numCases());
1496
LBlock* target = skipTrivialBlocks(mir->getCaseBlock(i))->lir();
1497
if (ObjectGroup* funcGroup = mir->getCaseObjectGroup(i)) {
1498
masm.branchTestObjGroupUnsafe(Assembler::Equal, input, funcGroup,
1499
target->label());
1500
} else {
1501
JSFunction* func = mir->getCase(i);
1502
masm.branchPtr(Assembler::Equal, input, ImmGCPtr(func), target->label());
1503
}
1504
}
1505
1506
// If at the end, and we have a fallback, we can jump to the fallback block.
1507
if (mir->hasFallback()) {
1508
masm.jump(skipTrivialBlocks(mir->getFallback())->lir()->label());
1509
return;
1510
}
1511
1512
// Otherwise, crash.
1513
masm.assumeUnreachable("Did not match input function!");
1514
}
1515
1516
void CodeGenerator::visitObjectGroupDispatch(LObjectGroupDispatch* lir) {
1517
MObjectGroupDispatch* mir = lir->mir();
1518
Register input = ToRegister(lir->input());
1519
Register temp = ToRegister(lir->temp());
1520
1521
// Load the incoming ObjectGroup in temp.
1522
masm.loadObjGroupUnsafe(input, temp);
1523
1524
// Compare ObjectGroups.
1525
MacroAssembler::BranchGCPtr lastBranch;
1526
LBlock* lastBlock = nullptr;
1527
InlinePropertyTable* propTable = mir->propTable();
1528
for (size_t i = 0; i < mir->numCases(); i++) {
1529
JSFunction* func = mir->getCase(i);
1530
LBlock* target = skipTrivialBlocks(mir->getCaseBlock(i))->lir();
1531
1532
DebugOnly<bool> found = false;
1533
// Find the function in the prop table.
1534
for (size_t j = 0; j < propTable->numEntries(); j++) {
1535
if (propTable->getFunction(j) != func) {
1536
continue;
1537
}
1538
1539
// Emit the previous prop's jump.
1540
if (lastBranch.isInitialized()) {
1541
lastBranch.emit(masm);
1542
}
1543
1544
// Setup jump for next iteration.
1545
ObjectGroup* group = propTable->getObjectGroup(j);
1546
lastBranch = MacroAssembler::BranchGCPtr(
1547
Assembler::Equal, temp, ImmGCPtr(group), target->label());
1548
lastBlock = target;
1549
found = true;
1550
}
1551
MOZ_ASSERT(found);
1552
}
1553
1554
// At this point the final case branch hasn't been emitted.
1555
1556
// Jump to fallback block if we have an unknown ObjectGroup. If there's no
1557
// fallback block, we should have handled all cases.
1558
if (!mir->hasFallback()) {
1559
MOZ_ASSERT(lastBranch.isInitialized());
1560
1561
Label ok;
1562
// Change the target of the branch to OK.
1563
lastBranch.relink(&ok);
1564
lastBranch.emit(masm);
1565
masm.assumeUnreachable("Unexpected ObjectGroup");
1566
masm.bind(&ok);
1567
1568
// If we don't naturally fall through to the target,
1569
// then jump to the target.
1570
if (!isNextBlock(lastBlock)) {
1571
masm.jump(lastBlock->label());
1572
}
1573
return;
1574
}
1575
1576
LBlock* fallback = skipTrivialBlocks(mir->getFallback())->lir();
1577
// This should only happen if we have zero cases. We're done then.
1578
if (!lastBranch.isInitialized()) {
1579
if (!isNextBlock(fallback)) {
1580
masm.jump(fallback->label());
1581
}
1582
return;
1583
}
1584
1585
// If we don't match the last object group and we have a fallback,
1586
// we should jump to it.
1587
lastBranch.invertCondition();
1588
lastBranch.relink(fallback->label());
1589
lastBranch.emit(masm);
1590
1591
if (!isNextBlock(lastBlock)) {
1592
masm.jump(lastBlock->label());
1593
}
1594
}
1595
1596
void CodeGenerator::visitBooleanToString(LBooleanToString* lir) {
1597
Register input = ToRegister(lir->input());
1598
Register output = ToRegister(lir->output());
1599
const JSAtomState& names = gen->runtime->names();
1600
Label true_, done;
1601
1602
masm.branchTest32(Assembler::NonZero, input, input, &true_);
1603
masm.movePtr(ImmGCPtr(names.false_), output);
1604
masm.jump(&done);
1605
1606
masm.bind(&