Source code

Revision control

Other Tools

1
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2
* vim: set ts=8 sts=2 et sw=2 tw=80:
3
* This Source Code Form is subject to the terms of the Mozilla Public
4
* License, v. 2.0. If a copy of the MPL was not distributed with this
5
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7
#include "jit/CodeGenerator.h"
8
9
#include "mozilla/Assertions.h"
10
#include "mozilla/Attributes.h"
11
#include "mozilla/Casting.h"
12
#include "mozilla/DebugOnly.h"
13
#include "mozilla/EnumeratedArray.h"
14
#include "mozilla/EnumeratedRange.h"
15
#include "mozilla/MathAlgorithms.h"
16
#include "mozilla/ScopeExit.h"
17
#include "mozilla/Unused.h"
18
19
#include <type_traits>
20
21
#include "jslibmath.h"
22
#include "jsmath.h"
23
#include "jsnum.h"
24
25
#include "builtin/Eval.h"
26
#include "builtin/MapObject.h"
27
#include "builtin/RegExp.h"
28
#include "builtin/SelfHostingDefines.h"
29
#include "builtin/String.h"
30
#include "builtin/TypedObject.h"
31
#include "gc/Nursery.h"
32
#include "irregexp/NativeRegExpMacroAssembler.h"
33
#include "jit/AtomicOperations.h"
34
#include "jit/BaselineCodeGen.h"
35
#include "jit/IonBuilder.h"
36
#include "jit/IonIC.h"
37
#include "jit/IonOptimizationLevels.h"
38
#include "jit/JitcodeMap.h"
39
#include "jit/JitSpewer.h"
40
#include "jit/Linker.h"
41
#include "jit/Lowering.h"
42
#include "jit/MIRGenerator.h"
43
#include "jit/MoveEmitter.h"
44
#include "jit/RangeAnalysis.h"
45
#include "jit/SharedICHelpers.h"
46
#include "jit/StackSlotAllocator.h"
47
#include "jit/VMFunctions.h"
48
#include "js/RegExpFlags.h" // JS::RegExpFlag
49
#include "util/Unicode.h"
50
#include "vm/AsyncFunction.h"
51
#include "vm/AsyncIteration.h"
52
#include "vm/EqualityOperations.h" // js::SameValue
53
#include "vm/MatchPairs.h"
54
#include "vm/RegExpObject.h"
55
#include "vm/RegExpStatics.h"
56
#include "vm/StringType.h"
57
#include "vm/TraceLogging.h"
58
#include "vm/TypedArrayObject.h"
59
#include "vtune/VTuneWrapper.h"
60
#include "wasm/WasmGC.h"
61
#include "wasm/WasmStubs.h"
62
63
#include "builtin/Boolean-inl.h"
64
#include "jit/MacroAssembler-inl.h"
65
#include "jit/shared/CodeGenerator-shared-inl.h"
66
#include "jit/shared/Lowering-shared-inl.h"
67
#include "jit/TemplateObject-inl.h"
68
#include "jit/VMFunctionList-inl.h"
69
#include "vm/Interpreter-inl.h"
70
#include "vm/JSScript-inl.h"
71
72
using namespace js;
73
using namespace js::jit;
74
75
using JS::GenericNaN;
76
using mozilla::AssertedCast;
77
using mozilla::DebugOnly;
78
using mozilla::FloatingPoint;
79
using mozilla::Maybe;
80
using mozilla::NegativeInfinity;
81
using mozilla::PositiveInfinity;
82
83
namespace js {
84
namespace jit {
85
86
#ifdef CHECK_OSIPOINT_REGISTERS
87
template <class Op>
88
static void HandleRegisterDump(Op op, MacroAssembler& masm,
89
LiveRegisterSet liveRegs, Register activation,
90
Register scratch) {
91
const size_t baseOffset = JitActivation::offsetOfRegs();
92
93
// Handle live GPRs.
94
for (GeneralRegisterIterator iter(liveRegs.gprs()); iter.more(); ++iter) {
95
Register reg = *iter;
96
Address dump(activation, baseOffset + RegisterDump::offsetOfRegister(reg));
97
98
if (reg == activation) {
99
// To use the original value of the activation register (that's
100
// now on top of the stack), we need the scratch register.
101
masm.push(scratch);
102
masm.loadPtr(Address(masm.getStackPointer(), sizeof(uintptr_t)), scratch);
103
op(scratch, dump);
104
masm.pop(scratch);
105
} else {
106
op(reg, dump);
107
}
108
}
109
110
// Handle live FPRs.
111
for (FloatRegisterIterator iter(liveRegs.fpus()); iter.more(); ++iter) {
112
FloatRegister reg = *iter;
113
Address dump(activation, baseOffset + RegisterDump::offsetOfRegister(reg));
114
op(reg, dump);
115
}
116
}
117
118
class StoreOp {
119
MacroAssembler& masm;
120
121
public:
122
explicit StoreOp(MacroAssembler& masm) : masm(masm) {}
123
124
void operator()(Register reg, Address dump) { masm.storePtr(reg, dump); }
125
void operator()(FloatRegister reg, Address dump) {
126
if (reg.isDouble()) {
127
masm.storeDouble(reg, dump);
128
} else if (reg.isSingle()) {
129
masm.storeFloat32(reg, dump);
130
# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
131
} else if (reg.isSimd128()) {
132
masm.storeUnalignedSimd128Float(reg, dump);
133
# endif
134
} else {
135
MOZ_CRASH("Unexpected register type.");
136
}
137
}
138
};
139
140
class VerifyOp {
141
MacroAssembler& masm;
142
Label* failure_;
143
144
public:
145
VerifyOp(MacroAssembler& masm, Label* failure)
146
: masm(masm), failure_(failure) {}
147
148
void operator()(Register reg, Address dump) {
149
masm.branchPtr(Assembler::NotEqual, dump, reg, failure_);
150
}
151
void operator()(FloatRegister reg, Address dump) {
152
if (reg.isDouble()) {
153
ScratchDoubleScope scratch(masm);
154
masm.loadDouble(dump, scratch);
155
masm.branchDouble(Assembler::DoubleNotEqual, scratch, reg, failure_);
156
} else if (reg.isSingle()) {
157
ScratchFloat32Scope scratch(masm);
158
masm.loadFloat32(dump, scratch);
159
masm.branchFloat(Assembler::DoubleNotEqual, scratch, reg, failure_);
160
}
161
162
// :TODO: (Bug 1133745) Add support to verify SIMD registers.
163
}
164
};
165
166
void CodeGenerator::verifyOsiPointRegs(LSafepoint* safepoint) {
167
// Ensure the live registers stored by callVM did not change between
168
// the call and this OsiPoint. Try-catch relies on this invariant.
169
170
// Load pointer to the JitActivation in a scratch register.
171
AllocatableGeneralRegisterSet allRegs(GeneralRegisterSet::All());
172
Register scratch = allRegs.takeAny();
173
masm.push(scratch);
174
masm.loadJitActivation(scratch);
175
176
// If we should not check registers (because the instruction did not call
177
// into the VM, or a GC happened), we're done.
178
Label failure, done;
179
Address checkRegs(scratch, JitActivation::offsetOfCheckRegs());
180
masm.branch32(Assembler::Equal, checkRegs, Imm32(0), &done);
181
182
// Having more than one VM function call made in one visit function at
183
// runtime is a sec-ciritcal error, because if we conservatively assume that
184
// one of the function call can re-enter Ion, then the invalidation process
185
// will potentially add a call at a random location, by patching the code
186
// before the return address.
187
masm.branch32(Assembler::NotEqual, checkRegs, Imm32(1), &failure);
188
189
// Set checkRegs to 0, so that we don't try to verify registers after we
190
// return from this script to the caller.
191
masm.store32(Imm32(0), checkRegs);
192
193
// Ignore clobbered registers. Some instructions (like LValueToInt32) modify
194
// temps after calling into the VM. This is fine because no other
195
// instructions (including this OsiPoint) will depend on them. Also
196
// backtracking can also use the same register for an input and an output.
197
// These are marked as clobbered and shouldn't get checked.
198
LiveRegisterSet liveRegs;
199
liveRegs.set() = RegisterSet::Intersect(
200
safepoint->liveRegs().set(),
201
RegisterSet::Not(safepoint->clobberedRegs().set()));
202
203
VerifyOp op(masm, &failure);
204
HandleRegisterDump<VerifyOp>(op, masm, liveRegs, scratch, allRegs.getAny());
205
206
masm.jump(&done);
207
208
// Do not profile the callWithABI that occurs below. This is to avoid a
209
// rare corner case that occurs when profiling interacts with itself:
210
//
211
// When slow profiling assertions are turned on, FunctionBoundary ops
212
// (which update the profiler pseudo-stack) may emit a callVM, which
213
// forces them to have an osi point associated with them. The
214
// FunctionBoundary for inline function entry is added to the caller's
215
// graph with a PC from the caller's code, but during codegen it modifies
216
// Gecko Profiler instrumentation to add the callee as the current top-most
217
// script. When codegen gets to the OSIPoint, and the callWithABI below is
218
// emitted, the codegen thinks that the current frame is the callee, but
219
// the PC it's using from the OSIPoint refers to the caller. This causes
220
// the profiler instrumentation of the callWithABI below to ASSERT, since
221
// the script and pc are mismatched. To avoid this, we simply omit
222
// instrumentation for these callWithABIs.
223
224
// Any live register captured by a safepoint (other than temp registers)
225
// must remain unchanged between the call and the OsiPoint instruction.
226
masm.bind(&failure);
227
masm.assumeUnreachable("Modified registers between VM call and OsiPoint");
228
229
masm.bind(&done);
230
masm.pop(scratch);
231
}
232
233
bool CodeGenerator::shouldVerifyOsiPointRegs(LSafepoint* safepoint) {
234
if (!checkOsiPointRegisters) {
235
return false;
236
}
237
238
if (safepoint->liveRegs().emptyGeneral() &&
239
safepoint->liveRegs().emptyFloat()) {
240
return false; // No registers to check.
241
}
242
243
return true;
244
}
245
246
void CodeGenerator::resetOsiPointRegs(LSafepoint* safepoint) {
247
if (!shouldVerifyOsiPointRegs(safepoint)) {
248
return;
249
}
250
251
// Set checkRegs to 0. If we perform a VM call, the instruction
252
// will set it to 1.
253
AllocatableGeneralRegisterSet allRegs(GeneralRegisterSet::All());
254
Register scratch = allRegs.takeAny();
255
masm.push(scratch);
256
masm.loadJitActivation(scratch);
257
Address checkRegs(scratch, JitActivation::offsetOfCheckRegs());
258
masm.store32(Imm32(0), checkRegs);
259
masm.pop(scratch);
260
}
261
262
static void StoreAllLiveRegs(MacroAssembler& masm, LiveRegisterSet liveRegs) {
263
// Store a copy of all live registers before performing the call.
264
// When we reach the OsiPoint, we can use this to check nothing
265
// modified them in the meantime.
266
267
// Load pointer to the JitActivation in a scratch register.
268
AllocatableGeneralRegisterSet allRegs(GeneralRegisterSet::All());
269
Register scratch = allRegs.takeAny();
270
masm.push(scratch);
271
masm.loadJitActivation(scratch);
272
273
Address checkRegs(scratch, JitActivation::offsetOfCheckRegs());
274
masm.add32(Imm32(1), checkRegs);
275
276
StoreOp op(masm);
277
HandleRegisterDump<StoreOp>(op, masm, liveRegs, scratch, allRegs.getAny());
278
279
masm.pop(scratch);
280
}
281
#endif // CHECK_OSIPOINT_REGISTERS
282
283
// Before doing any call to Cpp, you should ensure that volatile
284
// registers are evicted by the register allocator.
285
void CodeGenerator::callVMInternal(VMFunctionId id, LInstruction* ins,
286
const Register* dynStack) {
287
TrampolinePtr code = gen->jitRuntime()->getVMWrapper(id);
288
const VMFunctionData& fun = GetVMFunction(id);
289
290
#ifdef DEBUG
291
if (ins->mirRaw()) {
292
MOZ_ASSERT(ins->mirRaw()->isInstruction());
293
MInstruction* mir = ins->mirRaw()->toInstruction();
294
MOZ_ASSERT_IF(mir->needsResumePoint(), mir->resumePoint());
295
}
296
#endif
297
298
// Stack is:
299
// ... frame ...
300
// [args]
301
#ifdef DEBUG
302
MOZ_ASSERT(pushedArgs_ == fun.explicitArgs);
303
pushedArgs_ = 0;
304
#endif
305
306
#ifdef CHECK_OSIPOINT_REGISTERS
307
if (shouldVerifyOsiPointRegs(ins->safepoint())) {
308
StoreAllLiveRegs(masm, ins->safepoint()->liveRegs());
309
}
310
#endif
311
312
// Push an exit frame descriptor. If |dynStack| is a valid pointer to a
313
// register, then its value is added to the value of the |framePushed()| to
314
// fill the frame descriptor.
315
if (dynStack) {
316
masm.addPtr(Imm32(masm.framePushed()), *dynStack);
317
masm.makeFrameDescriptor(*dynStack, FrameType::IonJS,
318
ExitFrameLayout::Size());
319
masm.Push(*dynStack); // descriptor
320
} else {
321
masm.pushStaticFrameDescriptor(FrameType::IonJS, ExitFrameLayout::Size());
322
}
323
324
// Call the wrapper function. The wrapper is in charge to unwind the stack
325
// when returning from the call. Failures are handled with exceptions based
326
// on the return value of the C functions. To guard the outcome of the
327
// returned value, use another LIR instruction.
328
uint32_t callOffset = masm.callJit(code);
329
markSafepointAt(callOffset, ins);
330
331
// Remove rest of the frame left on the stack. We remove the return address
332
// which is implicitly poped when returning.
333
int framePop = sizeof(ExitFrameLayout) - sizeof(void*);
334
335
// Pop arguments from framePushed.
336
masm.implicitPop(fun.explicitStackSlots() * sizeof(void*) + framePop);
337
// Stack is:
338
// ... frame ...
339
}
340
341
template <typename Fn, Fn fn>
342
void CodeGenerator::callVM(LInstruction* ins, const Register* dynStack) {
343
VMFunctionId id = VMFunctionToId<Fn, fn>::id;
344
callVMInternal(id, ins, dynStack);
345
}
346
347
// ArgSeq store arguments for OutOfLineCallVM.
348
//
349
// OutOfLineCallVM are created with "oolCallVM" function. The third argument of
350
// this function is an instance of a class which provides a "generate" in charge
351
// of pushing the argument, with "pushArg", for a VMFunction.
352
//
353
// Such list of arguments can be created by using the "ArgList" function which
354
// creates one instance of "ArgSeq", where the type of the arguments are
355
// inferred from the type of the arguments.
356
//
357
// The list of arguments must be written in the same order as if you were
358
// calling the function in C++.
359
//
360
// Example:
361
// ArgList(ToRegister(lir->lhs()), ToRegister(lir->rhs()))
362
363
template <typename... ArgTypes>
364
class ArgSeq;
365
366
template <>
367
class ArgSeq<> {
368
public:
369
ArgSeq() {}
370
371
inline void generate(CodeGenerator* codegen) const {}
372
373
#ifdef DEBUG
374
static constexpr size_t numArgs = 0;
375
#endif
376
};
377
378
template <typename HeadType, typename... TailTypes>
379
class ArgSeq<HeadType, TailTypes...> : public ArgSeq<TailTypes...> {
380
private:
381
using RawHeadType = typename mozilla::RemoveReference<HeadType>::Type;
382
RawHeadType head_;
383
384
public:
385
template <typename ProvidedHead, typename... ProvidedTail>
386
explicit ArgSeq(ProvidedHead&& head, ProvidedTail&&... tail)
387
: ArgSeq<TailTypes...>(std::forward<ProvidedTail>(tail)...),
388
head_(std::forward<ProvidedHead>(head)) {}
389
390
// Arguments are pushed in reverse order, from last argument to first
391
// argument.
392
inline void generate(CodeGenerator* codegen) const {
393
this->ArgSeq<TailTypes...>::generate(codegen);
394
codegen->pushArg(head_);
395
}
396
397
#ifdef DEBUG
398
static constexpr size_t numArgs = sizeof...(TailTypes) + 1;
399
#endif
400
};
401
402
template <typename... ArgTypes>
403
inline ArgSeq<ArgTypes...> ArgList(ArgTypes&&... args) {
404
return ArgSeq<ArgTypes...>(std::forward<ArgTypes>(args)...);
405
}
406
407
// Store wrappers, to generate the right move of data after the VM call.
408
409
struct StoreNothing {
410
inline void generate(CodeGenerator* codegen) const {}
411
inline LiveRegisterSet clobbered() const {
412
return LiveRegisterSet(); // No register gets clobbered
413
}
414
};
415
416
class StoreRegisterTo {
417
private:
418
Register out_;
419
420
public:
421
explicit StoreRegisterTo(Register out) : out_(out) {}
422
423
inline void generate(CodeGenerator* codegen) const {
424
// It's okay to use storePointerResultTo here - the VMFunction wrapper
425
// ensures the upper bytes are zero for bool/int32 return values.
426
codegen->storePointerResultTo(out_);
427
}
428
inline LiveRegisterSet clobbered() const {
429
LiveRegisterSet set;
430
set.add(out_);
431
return set;
432
}
433
};
434
435
class StoreFloatRegisterTo {
436
private:
437
FloatRegister out_;
438
439
public:
440
explicit StoreFloatRegisterTo(FloatRegister out) : out_(out) {}
441
442
inline void generate(CodeGenerator* codegen) const {
443
codegen->storeFloatResultTo(out_);
444
}
445
inline LiveRegisterSet clobbered() const {
446
LiveRegisterSet set;
447
set.add(out_);
448
return set;
449
}
450
};
451
452
template <typename Output>
453
class StoreValueTo_ {
454
private:
455
Output out_;
456
457
public:
458
explicit StoreValueTo_(const Output& out) : out_(out) {}
459
460
inline void generate(CodeGenerator* codegen) const {
461
codegen->storeResultValueTo(out_);
462
}
463
inline LiveRegisterSet clobbered() const {
464
LiveRegisterSet set;
465
set.add(out_);
466
return set;
467
}
468
};
469
470
template <typename Output>
471
StoreValueTo_<Output> StoreValueTo(const Output& out) {
472
return StoreValueTo_<Output>(out);
473
}
474
475
template <typename Fn, Fn fn, class ArgSeq, class StoreOutputTo>
476
class OutOfLineCallVM : public OutOfLineCodeBase<CodeGenerator> {
477
private:
478
LInstruction* lir_;
479
ArgSeq args_;
480
StoreOutputTo out_;
481
482
public:
483
OutOfLineCallVM(LInstruction* lir, const ArgSeq& args,
484
const StoreOutputTo& out)
485
: lir_(lir), args_(args), out_(out) {}
486
487
void accept(CodeGenerator* codegen) override {
488
codegen->visitOutOfLineCallVM(this);
489
}
490
491
LInstruction* lir() const { return lir_; }
492
const ArgSeq& args() const { return args_; }
493
const StoreOutputTo& out() const { return out_; }
494
};
495
496
template <typename Fn, Fn fn, class ArgSeq, class StoreOutputTo>
497
OutOfLineCode* CodeGenerator::oolCallVM(LInstruction* lir, const ArgSeq& args,
498
const StoreOutputTo& out) {
499
MOZ_ASSERT(lir->mirRaw());
500
MOZ_ASSERT(lir->mirRaw()->isInstruction());
501
502
#ifdef DEBUG
503
VMFunctionId id = VMFunctionToId<Fn, fn>::id;
504
const VMFunctionData& fun = GetVMFunction(id);
505
MOZ_ASSERT(fun.explicitArgs == args.numArgs);
506
MOZ_ASSERT(fun.returnsData() !=
507
(mozilla::IsSame<StoreOutputTo, StoreNothing>::value));
508
#endif
509
510
OutOfLineCode* ool = new (alloc())
511
OutOfLineCallVM<Fn, fn, ArgSeq, StoreOutputTo>(lir, args, out);
512
addOutOfLineCode(ool, lir->mirRaw()->toInstruction());
513
return ool;
514
}
515
516
template <typename Fn, Fn fn, class ArgSeq, class StoreOutputTo>
517
void CodeGenerator::visitOutOfLineCallVM(
518
OutOfLineCallVM<Fn, fn, ArgSeq, StoreOutputTo>* ool) {
519
LInstruction* lir = ool->lir();
520
521
saveLive(lir);
522
ool->args().generate(this);
523
callVM<Fn, fn>(lir);
524
ool->out().generate(this);
525
restoreLiveIgnore(lir, ool->out().clobbered());
526
masm.jump(ool->rejoin());
527
}
528
529
class OutOfLineICFallback : public OutOfLineCodeBase<CodeGenerator> {
530
private:
531
LInstruction* lir_;
532
size_t cacheIndex_;
533
size_t cacheInfoIndex_;
534
535
public:
536
OutOfLineICFallback(LInstruction* lir, size_t cacheIndex,
537
size_t cacheInfoIndex)
538
: lir_(lir), cacheIndex_(cacheIndex), cacheInfoIndex_(cacheInfoIndex) {}
539
540
void bind(MacroAssembler* masm) override {
541
// The binding of the initial jump is done in
542
// CodeGenerator::visitOutOfLineICFallback.
543
}
544
545
size_t cacheIndex() const { return cacheIndex_; }
546
size_t cacheInfoIndex() const { return cacheInfoIndex_; }
547
LInstruction* lir() const { return lir_; }
548
549
void accept(CodeGenerator* codegen) override {
550
codegen->visitOutOfLineICFallback(this);
551
}
552
};
553
554
void CodeGeneratorShared::addIC(LInstruction* lir, size_t cacheIndex) {
555
if (cacheIndex == SIZE_MAX) {
556
masm.setOOM();
557
return;
558
}
559
560
DataPtr<IonIC> cache(this, cacheIndex);
561
MInstruction* mir = lir->mirRaw()->toInstruction();
562
if (mir->resumePoint()) {
563
cache->setScriptedLocation(mir->block()->info().script(),
564
mir->resumePoint()->pc());
565
} else {
566
cache->setIdempotent();
567
}
568
569
Register temp = cache->scratchRegisterForEntryJump();
570
icInfo_.back().icOffsetForJump = masm.movWithPatch(ImmWord(-1), temp);
571
masm.jump(Address(temp, 0));
572
573
MOZ_ASSERT(!icInfo_.empty());
574
575
OutOfLineICFallback* ool =
576
new (alloc()) OutOfLineICFallback(lir, cacheIndex, icInfo_.length() - 1);
577
addOutOfLineCode(ool, mir);
578
579
masm.bind(ool->rejoin());
580
cache->setRejoinOffset(CodeOffset(ool->rejoin()->offset()));
581
}
582
583
void CodeGenerator::visitOutOfLineICFallback(OutOfLineICFallback* ool) {
584
LInstruction* lir = ool->lir();
585
size_t cacheIndex = ool->cacheIndex();
586
size_t cacheInfoIndex = ool->cacheInfoIndex();
587
588
DataPtr<IonIC> ic(this, cacheIndex);
589
590
// Register the location of the OOL path in the IC.
591
ic->setFallbackOffset(CodeOffset(masm.currentOffset()));
592
593
switch (ic->kind()) {
594
case CacheKind::GetProp:
595
case CacheKind::GetElem: {
596
IonGetPropertyIC* getPropIC = ic->asGetPropertyIC();
597
598
saveLive(lir);
599
600
pushArg(getPropIC->id());
601
pushArg(getPropIC->value());
602
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
603
pushArg(ImmGCPtr(gen->info().script()));
604
605
using Fn = bool (*)(JSContext*, HandleScript, IonGetPropertyIC*,
606
HandleValue, HandleValue, MutableHandleValue);
607
callVM<Fn, IonGetPropertyIC::update>(lir);
608
609
StoreValueTo(getPropIC->output()).generate(this);
610
restoreLiveIgnore(lir, StoreValueTo(getPropIC->output()).clobbered());
611
612
masm.jump(ool->rejoin());
613
return;
614
}
615
case CacheKind::GetPropSuper:
616
case CacheKind::GetElemSuper: {
617
IonGetPropSuperIC* getPropSuperIC = ic->asGetPropSuperIC();
618
619
saveLive(lir);
620
621
pushArg(getPropSuperIC->id());
622
pushArg(getPropSuperIC->receiver());
623
pushArg(getPropSuperIC->object());
624
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
625
pushArg(ImmGCPtr(gen->info().script()));
626
627
using Fn =
628
bool (*)(JSContext*, HandleScript, IonGetPropSuperIC*, HandleObject,
629
HandleValue, HandleValue, MutableHandleValue);
630
callVM<Fn, IonGetPropSuperIC::update>(lir);
631
632
StoreValueTo(getPropSuperIC->output()).generate(this);
633
restoreLiveIgnore(lir,
634
StoreValueTo(getPropSuperIC->output()).clobbered());
635
636
masm.jump(ool->rejoin());
637
return;
638
}
639
case CacheKind::SetProp:
640
case CacheKind::SetElem: {
641
IonSetPropertyIC* setPropIC = ic->asSetPropertyIC();
642
643
saveLive(lir);
644
645
pushArg(setPropIC->rhs());
646
pushArg(setPropIC->id());
647
pushArg(setPropIC->object());
648
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
649
pushArg(ImmGCPtr(gen->info().script()));
650
651
using Fn = bool (*)(JSContext*, HandleScript, IonSetPropertyIC*,
652
HandleObject, HandleValue, HandleValue);
653
callVM<Fn, IonSetPropertyIC::update>(lir);
654
655
restoreLive(lir);
656
657
masm.jump(ool->rejoin());
658
return;
659
}
660
case CacheKind::GetName: {
661
IonGetNameIC* getNameIC = ic->asGetNameIC();
662
663
saveLive(lir);
664
665
pushArg(getNameIC->environment());
666
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
667
pushArg(ImmGCPtr(gen->info().script()));
668
669
using Fn = bool (*)(JSContext*, HandleScript, IonGetNameIC*, HandleObject,
670
MutableHandleValue);
671
callVM<Fn, IonGetNameIC::update>(lir);
672
673
StoreValueTo(getNameIC->output()).generate(this);
674
restoreLiveIgnore(lir, StoreValueTo(getNameIC->output()).clobbered());
675
676
masm.jump(ool->rejoin());
677
return;
678
}
679
case CacheKind::BindName: {
680
IonBindNameIC* bindNameIC = ic->asBindNameIC();
681
682
saveLive(lir);
683
684
pushArg(bindNameIC->environment());
685
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
686
pushArg(ImmGCPtr(gen->info().script()));
687
688
using Fn =
689
JSObject* (*)(JSContext*, HandleScript, IonBindNameIC*, HandleObject);
690
callVM<Fn, IonBindNameIC::update>(lir);
691
692
StoreRegisterTo(bindNameIC->output()).generate(this);
693
restoreLiveIgnore(lir, StoreRegisterTo(bindNameIC->output()).clobbered());
694
695
masm.jump(ool->rejoin());
696
return;
697
}
698
case CacheKind::GetIterator: {
699
IonGetIteratorIC* getIteratorIC = ic->asGetIteratorIC();
700
701
saveLive(lir);
702
703
pushArg(getIteratorIC->value());
704
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
705
pushArg(ImmGCPtr(gen->info().script()));
706
707
using Fn = JSObject* (*)(JSContext*, HandleScript, IonGetIteratorIC*,
708
HandleValue);
709
callVM<Fn, IonGetIteratorIC::update>(lir);
710
711
StoreRegisterTo(getIteratorIC->output()).generate(this);
712
restoreLiveIgnore(lir,
713
StoreRegisterTo(getIteratorIC->output()).clobbered());
714
715
masm.jump(ool->rejoin());
716
return;
717
}
718
case CacheKind::In: {
719
IonInIC* inIC = ic->asInIC();
720
721
saveLive(lir);
722
723
pushArg(inIC->object());
724
pushArg(inIC->key());
725
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
726
pushArg(ImmGCPtr(gen->info().script()));
727
728
using Fn = bool (*)(JSContext*, HandleScript, IonInIC*, HandleValue,
729
HandleObject, bool*);
730
callVM<Fn, IonInIC::update>(lir);
731
732
StoreRegisterTo(inIC->output()).generate(this);
733
restoreLiveIgnore(lir, StoreRegisterTo(inIC->output()).clobbered());
734
735
masm.jump(ool->rejoin());
736
return;
737
}
738
case CacheKind::HasOwn: {
739
IonHasOwnIC* hasOwnIC = ic->asHasOwnIC();
740
741
saveLive(lir);
742
743
pushArg(hasOwnIC->id());
744
pushArg(hasOwnIC->value());
745
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
746
pushArg(ImmGCPtr(gen->info().script()));
747
748
using Fn = bool (*)(JSContext*, HandleScript, IonHasOwnIC*, HandleValue,
749
HandleValue, int32_t*);
750
callVM<Fn, IonHasOwnIC::update>(lir);
751
752
StoreRegisterTo(hasOwnIC->output()).generate(this);
753
restoreLiveIgnore(lir, StoreRegisterTo(hasOwnIC->output()).clobbered());
754
755
masm.jump(ool->rejoin());
756
return;
757
}
758
case CacheKind::InstanceOf: {
759
IonInstanceOfIC* hasInstanceOfIC = ic->asInstanceOfIC();
760
761
saveLive(lir);
762
763
pushArg(hasInstanceOfIC->rhs());
764
pushArg(hasInstanceOfIC->lhs());
765
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
766
pushArg(ImmGCPtr(gen->info().script()));
767
768
using Fn = bool (*)(JSContext*, HandleScript, IonInstanceOfIC*,
769
HandleValue lhs, HandleObject rhs, bool* res);
770
callVM<Fn, IonInstanceOfIC::update>(lir);
771
772
StoreRegisterTo(hasInstanceOfIC->output()).generate(this);
773
restoreLiveIgnore(lir,
774
StoreRegisterTo(hasInstanceOfIC->output()).clobbered());
775
776
masm.jump(ool->rejoin());
777
return;
778
}
779
case CacheKind::UnaryArith: {
780
IonUnaryArithIC* unaryArithIC = ic->asUnaryArithIC();
781
782
saveLive(lir);
783
784
pushArg(unaryArithIC->input());
785
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
786
pushArg(ImmGCPtr(gen->info().script()));
787
788
using Fn = bool (*)(JSContext * cx, HandleScript outerScript,
789
IonUnaryArithIC * stub, HandleValue val,
790
MutableHandleValue res);
791
callVM<Fn, IonUnaryArithIC::update>(lir);
792
793
StoreValueTo(unaryArithIC->output()).generate(this);
794
restoreLiveIgnore(lir, StoreValueTo(unaryArithIC->output()).clobbered());
795
796
masm.jump(ool->rejoin());
797
return;
798
}
799
case CacheKind::BinaryArith: {
800
IonBinaryArithIC* binaryArithIC = ic->asBinaryArithIC();
801
802
saveLive(lir);
803
804
pushArg(binaryArithIC->rhs());
805
pushArg(binaryArithIC->lhs());
806
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
807
pushArg(ImmGCPtr(gen->info().script()));
808
809
using Fn = bool (*)(JSContext * cx, HandleScript outerScript,
810
IonBinaryArithIC * stub, HandleValue lhs,
811
HandleValue rhs, MutableHandleValue res);
812
callVM<Fn, IonBinaryArithIC::update>(lir);
813
814
StoreValueTo(binaryArithIC->output()).generate(this);
815
restoreLiveIgnore(lir, StoreValueTo(binaryArithIC->output()).clobbered());
816
817
masm.jump(ool->rejoin());
818
return;
819
}
820
case CacheKind::Compare: {
821
IonCompareIC* compareIC = ic->asCompareIC();
822
823
saveLive(lir);
824
825
pushArg(compareIC->rhs());
826
pushArg(compareIC->lhs());
827
icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
828
pushArg(ImmGCPtr(gen->info().script()));
829
830
using Fn = bool (*)(JSContext * cx, HandleScript outerScript,
831
IonCompareIC * stub, HandleValue lhs, HandleValue rhs,
832
bool* res);
833
callVM<Fn, IonCompareIC::update>(lir);
834
835
StoreRegisterTo(compareIC->output()).generate(this);
836
restoreLiveIgnore(lir, StoreRegisterTo(compareIC->output()).clobbered());
837
838
masm.jump(ool->rejoin());
839
return;
840
}
841
case CacheKind::Call:
842
case CacheKind::TypeOf:
843
case CacheKind::ToBool:
844
case CacheKind::GetIntrinsic:
845
case CacheKind::NewObject:
846
MOZ_CRASH("Unsupported IC");
847
}
848
MOZ_CRASH();
849
}
850
851
StringObject* MNewStringObject::templateObj() const {
852
return &templateObj_->as<StringObject>();
853
}
854
855
CodeGenerator::CodeGenerator(MIRGenerator* gen, LIRGraph* graph,
856
MacroAssembler* masm)
857
: CodeGeneratorSpecific(gen, graph, masm),
858
ionScriptLabels_(gen->alloc()),
859
scriptCounts_(nullptr),
860
realmStubsToReadBarrier_(0) {}
861
862
CodeGenerator::~CodeGenerator() { js_delete(scriptCounts_); }
863
864
void CodeGenerator::visitValueToInt32(LValueToInt32* lir) {
865
ValueOperand operand = ToValue(lir, LValueToInt32::Input);
866
Register output = ToRegister(lir->output());
867
FloatRegister temp = ToFloatRegister(lir->tempFloat());
868
869
MDefinition* input;
870
if (lir->mode() == LValueToInt32::NORMAL) {
871
input = lir->mirNormal()->input();
872
} else {
873
input = lir->mirTruncate()->input();
874
}
875
876
Label fails;
877
if (lir->mode() == LValueToInt32::TRUNCATE) {
878
OutOfLineCode* oolDouble = oolTruncateDouble(temp, output, lir->mir());
879
880
// We can only handle strings in truncation contexts, like bitwise
881
// operations.
882
Label* stringEntry;
883
Label* stringRejoin;
884
Register stringReg;
885
if (input->mightBeType(MIRType::String)) {
886
stringReg = ToRegister(lir->temp());
887
using Fn = bool (*)(JSContext*, JSString*, double*);
888
OutOfLineCode* oolString = oolCallVM<Fn, StringToNumber>(
889
lir, ArgList(stringReg), StoreFloatRegisterTo(temp));
890
stringEntry = oolString->entry();
891
stringRejoin = oolString->rejoin();
892
} else {
893
stringReg = InvalidReg;
894
stringEntry = nullptr;
895
stringRejoin = nullptr;
896
}
897
898
masm.truncateValueToInt32(operand, input, stringEntry, stringRejoin,
899
oolDouble->entry(), stringReg, temp, output,
900
&fails);
901
masm.bind(oolDouble->rejoin());
902
} else {
903
masm.convertValueToInt32(operand, input, temp, output, &fails,
904
lir->mirNormal()->canBeNegativeZero(),
905
lir->mirNormal()->conversion());
906
}
907
908
bailoutFrom(&fails, lir->snapshot());
909
}
910
911
void CodeGenerator::visitValueToDouble(LValueToDouble* lir) {
912
MToDouble* mir = lir->mir();
913
ValueOperand operand = ToValue(lir, LValueToDouble::Input);
914
FloatRegister output = ToFloatRegister(lir->output());
915
916
Label isDouble, isInt32, isBool, isNull, isUndefined, done;
917
bool hasBoolean = false, hasNull = false, hasUndefined = false;
918
919
{
920
ScratchTagScope tag(masm, operand);
921
masm.splitTagForTest(operand, tag);
922
923
masm.branchTestDouble(Assembler::Equal, tag, &isDouble);
924
masm.branchTestInt32(Assembler::Equal, tag, &isInt32);
925
926
if (mir->conversion() != MToFPInstruction::NumbersOnly) {
927
masm.branchTestBoolean(Assembler::Equal, tag, &isBool);
928
masm.branchTestUndefined(Assembler::Equal, tag, &isUndefined);
929
hasBoolean = true;
930
hasUndefined = true;
931
if (mir->conversion() != MToFPInstruction::NonNullNonStringPrimitives) {
932
masm.branchTestNull(Assembler::Equal, tag, &isNull);
933
hasNull = true;
934
}
935
}
936
}
937
938
bailout(lir->snapshot());
939
940
if (hasNull) {
941
masm.bind(&isNull);
942
masm.loadConstantDouble(0.0, output);
943
masm.jump(&done);
944
}
945
946
if (hasUndefined) {
947
masm.bind(&isUndefined);
948
masm.loadConstantDouble(GenericNaN(), output);
949
masm.jump(&done);
950
}
951
952
if (hasBoolean) {
953
masm.bind(&isBool);
954
masm.boolValueToDouble(operand, output);
955
masm.jump(&done);
956
}
957
958
masm.bind(&isInt32);
959
masm.int32ValueToDouble(operand, output);
960
masm.jump(&done);
961
962
masm.bind(&isDouble);
963
masm.unboxDouble(operand, output);
964
masm.bind(&done);
965
}
966
967
void CodeGenerator::visitValueToFloat32(LValueToFloat32* lir) {
968
MToFloat32* mir = lir->mir();
969
ValueOperand operand = ToValue(lir, LValueToFloat32::Input);
970
FloatRegister output = ToFloatRegister(lir->output());
971
972
Label isDouble, isInt32, isBool, isNull, isUndefined, done;
973
bool hasBoolean = false, hasNull = false, hasUndefined = false;
974
975
{
976
ScratchTagScope tag(masm, operand);
977
masm.splitTagForTest(operand, tag);
978
979
masm.branchTestDouble(Assembler::Equal, tag, &isDouble);
980
masm.branchTestInt32(Assembler::Equal, tag, &isInt32);
981
982
if (mir->conversion() != MToFPInstruction::NumbersOnly) {
983
masm.branchTestBoolean(Assembler::Equal, tag, &isBool);
984
masm.branchTestUndefined(Assembler::Equal, tag, &isUndefined);
985
hasBoolean = true;
986
hasUndefined = true;
987
if (mir->conversion() != MToFPInstruction::NonNullNonStringPrimitives) {
988
masm.branchTestNull(Assembler::Equal, tag, &isNull);
989
hasNull = true;
990
}
991
}
992
}
993
994
bailout(lir->snapshot());
995
996
if (hasNull) {
997
masm.bind(&isNull);
998
masm.loadConstantFloat32(0.0f, output);
999
masm.jump(&done);
1000
}
1001
1002
if (hasUndefined) {
1003
masm.bind(&isUndefined);
1004
masm.loadConstantFloat32(float(GenericNaN()), output);
1005
masm.jump(&done);
1006
}
1007
1008
if (hasBoolean) {
1009
masm.bind(&isBool);
1010
masm.boolValueToFloat32(operand, output);
1011
masm.jump(&done);
1012
}
1013
1014
masm.bind(&isInt32);
1015
masm.int32ValueToFloat32(operand, output);
1016
masm.jump(&done);
1017
1018
masm.bind(&isDouble);
1019
// ARM and MIPS may not have a double register available if we've
1020
// allocated output as a float32.
1021
#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS32)
1022
ScratchDoubleScope fpscratch(masm);
1023
masm.unboxDouble(operand, fpscratch);
1024
masm.convertDoubleToFloat32(fpscratch, output);
1025
#else
1026
masm.unboxDouble(operand, output);
1027
masm.convertDoubleToFloat32(output, output);
1028
#endif
1029
masm.bind(&done);
1030
}
1031
1032
void CodeGenerator::visitInt32ToDouble(LInt32ToDouble* lir) {
1033
masm.convertInt32ToDouble(ToRegister(lir->input()),
1034
ToFloatRegister(lir->output()));
1035
}
1036
1037
void CodeGenerator::visitFloat32ToDouble(LFloat32ToDouble* lir) {
1038
masm.convertFloat32ToDouble(ToFloatRegister(lir->input()),
1039
ToFloatRegister(lir->output()));
1040
}
1041
1042
void CodeGenerator::visitDoubleToFloat32(LDoubleToFloat32* lir) {
1043
masm.convertDoubleToFloat32(ToFloatRegister(lir->input()),
1044
ToFloatRegister(lir->output()));
1045
}
1046
1047
void CodeGenerator::visitInt32ToFloat32(LInt32ToFloat32* lir) {
1048
masm.convertInt32ToFloat32(ToRegister(lir->input()),
1049
ToFloatRegister(lir->output()));
1050
}
1051
1052
void CodeGenerator::visitDoubleToInt32(LDoubleToInt32* lir) {
1053
Label fail;
1054
FloatRegister input = ToFloatRegister(lir->input());
1055
Register output = ToRegister(lir->output());
1056
masm.convertDoubleToInt32(input, output, &fail,
1057
lir->mir()->canBeNegativeZero());
1058
bailoutFrom(&fail, lir->snapshot());
1059
}
1060
1061
void CodeGenerator::visitFloat32ToInt32(LFloat32ToInt32* lir) {
1062
Label fail;
1063
FloatRegister input = ToFloatRegister(lir->input());
1064
Register output = ToRegister(lir->output());
1065
masm.convertFloat32ToInt32(input, output, &fail,
1066
lir->mir()->canBeNegativeZero());
1067
bailoutFrom(&fail, lir->snapshot());
1068
}
1069
1070
void CodeGenerator::emitOOLTestObject(Register objreg,
1071
Label* ifEmulatesUndefined,
1072
Label* ifDoesntEmulateUndefined,
1073
Register scratch) {
1074
saveVolatile(scratch);
1075
masm.setupUnalignedABICall(scratch);
1076
masm.passABIArg(objreg);
1077
masm.callWithABI(JS_FUNC_TO_DATA_PTR(void*, js::EmulatesUndefined));
1078
masm.storeCallBoolResult(scratch);
1079
restoreVolatile(scratch);
1080
1081
masm.branchIfTrueBool(scratch, ifEmulatesUndefined);
1082
masm.jump(ifDoesntEmulateUndefined);
1083
}
1084
1085
// Base out-of-line code generator for all tests of the truthiness of an
1086
// object, where the object might not be truthy. (Recall that per spec all
1087
// objects are truthy, but we implement the JSCLASS_EMULATES_UNDEFINED class
1088
// flag to permit objects to look like |undefined| in certain contexts,
1089
// including in object truthiness testing.) We check truthiness inline except
1090
// when we're testing it on a proxy (or if TI guarantees us that the specified
1091
// object will never emulate |undefined|), in which case out-of-line code will
1092
// call EmulatesUndefined for a conclusive answer.
1093
class OutOfLineTestObject : public OutOfLineCodeBase<CodeGenerator> {
1094
Register objreg_;
1095
Register scratch_;
1096
1097
Label* ifEmulatesUndefined_;
1098
Label* ifDoesntEmulateUndefined_;
1099
1100
#ifdef DEBUG
1101
bool initialized() { return ifEmulatesUndefined_ != nullptr; }
1102
#endif
1103
1104
public:
1105
OutOfLineTestObject()
1106
: ifEmulatesUndefined_(nullptr), ifDoesntEmulateUndefined_(nullptr) {}
1107
1108
void accept(CodeGenerator* codegen) final {
1109
MOZ_ASSERT(initialized());
1110
codegen->emitOOLTestObject(objreg_, ifEmulatesUndefined_,
1111
ifDoesntEmulateUndefined_, scratch_);
1112
}
1113
1114
// Specify the register where the object to be tested is found, labels to
1115
// jump to if the object is truthy or falsy, and a scratch register for
1116
// use in the out-of-line path.
1117
void setInputAndTargets(Register objreg, Label* ifEmulatesUndefined,
1118
Label* ifDoesntEmulateUndefined, Register scratch) {
1119
MOZ_ASSERT(!initialized());
1120
MOZ_ASSERT(ifEmulatesUndefined);
1121
objreg_ = objreg;
1122
scratch_ = scratch;
1123
ifEmulatesUndefined_ = ifEmulatesUndefined;
1124
ifDoesntEmulateUndefined_ = ifDoesntEmulateUndefined;
1125
}
1126
};
1127
1128
// A subclass of OutOfLineTestObject containing two extra labels, for use when
1129
// the ifTruthy/ifFalsy labels are needed in inline code as well as out-of-line
1130
// code. The user should bind these labels in inline code, and specify them as
1131
// targets via setInputAndTargets, as appropriate.
1132
class OutOfLineTestObjectWithLabels : public OutOfLineTestObject {
1133
Label label1_;
1134
Label label2_;
1135
1136
public:
1137
OutOfLineTestObjectWithLabels() {}
1138
1139
Label* label1() { return &label1_; }
1140
Label* label2() { return &label2_; }
1141
};
1142
1143
void CodeGenerator::testObjectEmulatesUndefinedKernel(
1144
Register objreg, Label* ifEmulatesUndefined,
1145
Label* ifDoesntEmulateUndefined, Register scratch,
1146
OutOfLineTestObject* ool) {
1147
ool->setInputAndTargets(objreg, ifEmulatesUndefined, ifDoesntEmulateUndefined,
1148
scratch);
1149
1150
// Perform a fast-path check of the object's class flags if the object's
1151
// not a proxy. Let out-of-line code handle the slow cases that require
1152
// saving registers, making a function call, and restoring registers.
1153
masm.branchIfObjectEmulatesUndefined(objreg, scratch, ool->entry(),
1154
ifEmulatesUndefined);
1155
}
1156
1157
void CodeGenerator::branchTestObjectEmulatesUndefined(
1158
Register objreg, Label* ifEmulatesUndefined,
1159
Label* ifDoesntEmulateUndefined, Register scratch,
1160
OutOfLineTestObject* ool) {
1161
MOZ_ASSERT(!ifDoesntEmulateUndefined->bound(),
1162
"ifDoesntEmulateUndefined will be bound to the fallthrough path");
1163
1164
testObjectEmulatesUndefinedKernel(objreg, ifEmulatesUndefined,
1165
ifDoesntEmulateUndefined, scratch, ool);
1166
masm.bind(ifDoesntEmulateUndefined);
1167
}
1168
1169
void CodeGenerator::testObjectEmulatesUndefined(Register objreg,
1170
Label* ifEmulatesUndefined,
1171
Label* ifDoesntEmulateUndefined,
1172
Register scratch,
1173
OutOfLineTestObject* ool) {
1174
testObjectEmulatesUndefinedKernel(objreg, ifEmulatesUndefined,
1175
ifDoesntEmulateUndefined, scratch, ool);
1176
masm.jump(ifDoesntEmulateUndefined);
1177
}
1178
1179
void CodeGenerator::testValueTruthyKernel(
1180
const ValueOperand& value, const LDefinition* scratch1,
1181
const LDefinition* scratch2, FloatRegister fr, Label* ifTruthy,
1182
Label* ifFalsy, OutOfLineTestObject* ool, MDefinition* valueMIR) {
1183
// Count the number of possible type tags we might have, so we'll know when
1184
// we've checked them all and hence can avoid emitting a tag check for the
1185
// last one. In particular, whenever tagCount is 1 that means we've tried
1186
// all but one of them already so we know exactly what's left based on the
1187
// mightBe* booleans.
1188
bool mightBeUndefined = valueMIR->mightBeType(MIRType::Undefined);
1189
bool mightBeNull = valueMIR->mightBeType(MIRType::Null);
1190
bool mightBeBoolean = valueMIR->mightBeType(MIRType::Boolean);
1191
bool mightBeInt32 = valueMIR->mightBeType(MIRType::Int32);
1192
bool mightBeObject = valueMIR->mightBeType(MIRType::Object);
1193
bool mightBeString = valueMIR->mightBeType(MIRType::String);
1194
bool mightBeSymbol = valueMIR->mightBeType(MIRType::Symbol);
1195
bool mightBeDouble = valueMIR->mightBeType(MIRType::Double);
1196
bool mightBeBigInt = valueMIR->mightBeType(MIRType::BigInt);
1197
int tagCount = int(mightBeUndefined) + int(mightBeNull) +
1198
int(mightBeBoolean) + int(mightBeInt32) + int(mightBeObject) +
1199
int(mightBeString) + int(mightBeSymbol) + int(mightBeDouble) +
1200
int(mightBeBigInt);
1201
1202
MOZ_ASSERT_IF(!valueMIR->emptyResultTypeSet(), tagCount > 0);
1203
1204
// If we know we're null or undefined, we're definitely falsy, no
1205
// need to even check the tag.
1206
if (int(mightBeNull) + int(mightBeUndefined) == tagCount) {
1207
masm.jump(ifFalsy);
1208
return;
1209
}
1210
1211
ScratchTagScope tag(masm, value);
1212
masm.splitTagForTest(value, tag);
1213
1214
if (mightBeUndefined) {
1215
MOZ_ASSERT(tagCount > 1);
1216
masm.branchTestUndefined(Assembler::Equal, tag, ifFalsy);
1217
--tagCount;
1218
}
1219
1220
if (mightBeNull) {
1221
MOZ_ASSERT(tagCount > 1);
1222
masm.branchTestNull(Assembler::Equal, tag, ifFalsy);
1223
--tagCount;
1224
}
1225
1226
if (mightBeBoolean) {
1227
MOZ_ASSERT(tagCount != 0);
1228
Label notBoolean;
1229
if (tagCount != 1) {
1230
masm.branchTestBoolean(Assembler::NotEqual, tag, &notBoolean);
1231
}
1232
{
1233
ScratchTagScopeRelease _(&tag);
1234
masm.branchTestBooleanTruthy(false, value, ifFalsy);
1235
}
1236
if (tagCount != 1) {
1237
masm.jump(ifTruthy);
1238
}
1239
// Else just fall through to truthiness.
1240
masm.bind(&notBoolean);
1241
--tagCount;
1242
}
1243
1244
if (mightBeInt32) {
1245
MOZ_ASSERT(tagCount != 0);
1246
Label notInt32;
1247
if (tagCount != 1) {
1248
masm.branchTestInt32(Assembler::NotEqual, tag, &notInt32);
1249
}
1250
{
1251
ScratchTagScopeRelease _(&tag);
1252
masm.branchTestInt32Truthy(false, value, ifFalsy);
1253
}
1254
if (tagCount != 1) {
1255
masm.jump(ifTruthy);
1256
}
1257
// Else just fall through to truthiness.
1258
masm.bind(&notInt32);
1259
--tagCount;
1260
}
1261
1262
if (mightBeObject) {
1263
MOZ_ASSERT(tagCount != 0);
1264
if (ool) {
1265
Label notObject;
1266
1267
if (tagCount != 1) {
1268
masm.branchTestObject(Assembler::NotEqual, tag, &notObject);
1269
}
1270
1271
{
1272
ScratchTagScopeRelease _(&tag);
1273
Register objreg = masm.extractObject(value, ToRegister(scratch1));
1274
testObjectEmulatesUndefined(objreg, ifFalsy, ifTruthy,
1275
ToRegister(scratch2), ool);
1276
}
1277
1278
masm.bind(&notObject);
1279
} else {
1280
if (tagCount != 1) {
1281
masm.branchTestObject(Assembler::Equal, tag, ifTruthy);
1282
}
1283
// Else just fall through to truthiness.
1284
}
1285
--tagCount;
1286
} else {
1287
MOZ_ASSERT(!ool,
1288
"We better not have an unused OOL path, since the code "
1289
"generator will try to "
1290
"generate code for it but we never set up its labels, which "
1291
"will cause null "
1292
"derefs of those labels.");
1293
}
1294
1295
if (mightBeString) {
1296
// Test if a string is non-empty.
1297
MOZ_ASSERT(tagCount != 0);
1298
Label notString;
1299
if (tagCount != 1) {
1300
masm.branchTestString(Assembler::NotEqual, tag, &notString);
1301
}
1302
{
1303
ScratchTagScopeRelease _(&tag);
1304
masm.branchTestStringTruthy(false, value, ifFalsy);
1305
}
1306
if (tagCount != 1) {
1307
masm.jump(ifTruthy);
1308
}
1309
// Else just fall through to truthiness.
1310
masm.bind(&notString);
1311
--tagCount;
1312
}
1313
1314
if (mightBeBigInt) {
1315
MOZ_ASSERT(tagCount != 0);
1316
Label notBigInt;
1317
if (tagCount != 1) {
1318
masm.branchTestBigInt(Assembler::NotEqual, tag, &notBigInt);
1319
}
1320
{
1321
ScratchTagScopeRelease _(&tag);
1322
masm.branchTestBigIntTruthy(false, value, ifFalsy);
1323
}
1324
if (tagCount != 1) {
1325
masm.jump(ifTruthy);
1326
}
1327
masm.bind(&notBigInt);
1328
--tagCount;
1329
}
1330
1331
if (mightBeSymbol) {
1332
// All symbols are truthy.
1333
MOZ_ASSERT(tagCount != 0);
1334
if (tagCount != 1) {
1335
masm.branchTestSymbol(Assembler::Equal, tag, ifTruthy);
1336
}
1337
// Else fall through to ifTruthy.
1338
--tagCount;
1339
}
1340
1341
if (mightBeDouble) {
1342
MOZ_ASSERT(tagCount == 1);
1343
// If we reach here the value is a double.
1344
{
1345
ScratchTagScopeRelease _(&tag);
1346
masm.unboxDouble(value, fr);
1347
masm.branchTestDoubleTruthy(false, fr, ifFalsy);
1348
}
1349
--tagCount;
1350
}
1351
1352
MOZ_ASSERT(tagCount == 0);
1353
1354
// Fall through for truthy.
1355
}
1356
1357
void CodeGenerator::testValueTruthy(const ValueOperand& value,
1358
const LDefinition* scratch1,
1359
const LDefinition* scratch2,
1360
FloatRegister fr, Label* ifTruthy,
1361
Label* ifFalsy, OutOfLineTestObject* ool,
1362
MDefinition* valueMIR) {
1363
testValueTruthyKernel(value, scratch1, scratch2, fr, ifTruthy, ifFalsy, ool,
1364
valueMIR);
1365
masm.jump(ifTruthy);
1366
}
1367
1368
void CodeGenerator::visitTestOAndBranch(LTestOAndBranch* lir) {
1369
MIRType inputType = lir->mir()->input()->type();
1370
MOZ_ASSERT(inputType == MIRType::ObjectOrNull ||
1371
lir->mir()->operandMightEmulateUndefined(),
1372
"If the object couldn't emulate undefined, this should have been "
1373
"folded.");
1374
1375
Label* truthy = getJumpLabelForBranch(lir->ifTruthy());
1376
Label* falsy = getJumpLabelForBranch(lir->ifFalsy());
1377
Register input = ToRegister(lir->input());
1378
1379
if (lir->mir()->operandMightEmulateUndefined()) {
1380
if (inputType == MIRType::ObjectOrNull) {
1381
masm.branchTestPtr(Assembler::Zero, input, input, falsy);
1382
}
1383
1384
OutOfLineTestObject* ool = new (alloc()) OutOfLineTestObject();
1385
addOutOfLineCode(ool, lir->mir());
1386
1387
testObjectEmulatesUndefined(input, falsy, truthy, ToRegister(lir->temp()),
1388
ool);
1389
} else {
1390
MOZ_ASSERT(inputType == MIRType::ObjectOrNull);
1391
testZeroEmitBranch(Assembler::NotEqual, input, lir->ifTruthy(),
1392
lir->ifFalsy());
1393
}
1394
}
1395
1396
void CodeGenerator::visitTestVAndBranch(LTestVAndBranch* lir) {
1397
OutOfLineTestObject* ool = nullptr;
1398
MDefinition* input = lir->mir()->input();
1399
// Unfortunately, it's possible that someone (e.g. phi elimination) switched
1400
// out our input after we did cacheOperandMightEmulateUndefined. So we
1401
// might think it can emulate undefined _and_ know that it can't be an
1402
// object.
1403
if (lir->mir()->operandMightEmulateUndefined() &&
1404
input->mightBeType(MIRType::Object)) {
1405
ool = new (alloc()) OutOfLineTestObject();
1406
addOutOfLineCode(ool, lir->mir());
1407
}
1408
1409
Label* truthy = getJumpLabelForBranch(lir->ifTruthy());
1410
Label* falsy = getJumpLabelForBranch(lir->ifFalsy());
1411
1412
testValueTruthy(ToValue(lir, LTestVAndBranch::Input), lir->temp1(),
1413
lir->temp2(), ToFloatRegister(lir->tempFloat()), truthy,
1414
falsy, ool, input);
1415
}
1416
1417
void CodeGenerator::visitFunctionDispatch(LFunctionDispatch* lir) {
1418
MFunctionDispatch* mir = lir->mir();
1419
Register input = ToRegister(lir->input());
1420
1421
// Compare function pointers
1422
for (size_t i = 0; i < mir->numCases(); i++) {
1423
MOZ_ASSERT(i < mir->numCases());
1424
LBlock* target = skipTrivialBlocks(mir->getCaseBlock(i))->lir();
1425
if (ObjectGroup* funcGroup = mir->getCaseObjectGroup(i)) {
1426
masm.branchTestObjGroupUnsafe(Assembler::Equal, input, funcGroup,
1427
target->label());
1428
} else {
1429
JSFunction* func = mir->getCase(i);
1430
masm.branchPtr(Assembler::Equal, input, ImmGCPtr(func), target->label());
1431
}
1432
}
1433
1434
// If at the end, and we have a fallback, we can jump to the fallback block.
1435
if (mir->hasFallback()) {
1436
masm.jump(skipTrivialBlocks(mir->getFallback())->lir()->label());
1437
return;
1438
}
1439
1440
// Otherwise, crash.
1441
masm.assumeUnreachable("Did not match input function!");
1442
}
1443
1444
void CodeGenerator::visitObjectGroupDispatch(LObjectGroupDispatch* lir) {
1445
MObjectGroupDispatch* mir = lir->mir();
1446
Register input = ToRegister(lir->input());
1447
Register temp = ToRegister(lir->temp());
1448
1449
// Load the incoming ObjectGroup in temp.
1450
masm.loadObjGroupUnsafe(input, temp);
1451
1452
// Compare ObjectGroups.
1453
MacroAssembler::BranchGCPtr lastBranch;
1454
LBlock* lastBlock = nullptr;
1455
InlinePropertyTable* propTable = mir->propTable();
1456
for (size_t i = 0; i < mir->numCases(); i++) {
1457
JSFunction* func = mir->getCase(i);
1458
LBlock* target = skipTrivialBlocks(mir->getCaseBlock(i))->lir();
1459
1460
DebugOnly<bool> found = false;
1461
// Find the function in the prop table.
1462
for (size_t j = 0; j < propTable->numEntries(); j++) {
1463
if (propTable->getFunction(j) != func) {
1464
continue;
1465
}
1466
1467
// Emit the previous prop's jump.
1468
if (lastBranch.isInitialized()) {
1469
lastBranch.emit(masm);
1470
}
1471
1472
// Setup jump for next iteration.
1473
ObjectGroup* group = propTable->getObjectGroup(j);
1474
lastBranch = MacroAssembler::BranchGCPtr(
1475
Assembler::Equal, temp, ImmGCPtr(group), target->label());
1476
lastBlock = target;
1477
found = true;
1478
}
1479
MOZ_ASSERT(found);
1480
}
1481
1482
// At this point the final case branch hasn't been emitted.
1483
1484
// Jump to fallback block if we have an unknown ObjectGroup. If there's no
1485
// fallback block, we should have handled all cases.
1486
if (!mir->hasFallback()) {
1487
MOZ_ASSERT(lastBranch.isInitialized());
1488
1489
Label ok;
1490
// Change the target of the branch to OK.
1491
lastBranch.relink(&ok);
1492
lastBranch.emit(masm);
1493
masm.assumeUnreachable("Unexpected ObjectGroup");
1494
masm.bind(&ok);
1495
1496
// If we don't naturally fall through to the target,
1497
// then jump to the target.
1498
if (!isNextBlock(lastBlock)) {
1499
masm.jump(lastBlock->label());
1500
}
1501
return;
1502
}
1503
1504
LBlock* fallback = skipTrivialBlocks(mir->getFallback())->lir();
1505
// This should only happen if we have zero cases. We're done then.
1506
if (!lastBranch.isInitialized()) {
1507
if (!isNextBlock(fallback)) {
1508
masm.jump(fallback->label());
1509
}
1510
return;
1511
}
1512
1513
// If we don't match the last object group and we have a fallback,
1514
// we should jump to it.
1515
lastBranch.invertCondition();
1516
lastBranch.relink(fallback->label());
1517
lastBranch.emit(masm);
1518
1519
if (!isNextBlock(lastBlock)) {
1520
masm.jump(lastBlock->label());
1521
}
1522
}
1523
1524
void CodeGenerator::visitBooleanToString(LBooleanToString* lir) {
1525
Register input = ToRegister(lir->input());
1526
Register output = ToRegister(lir->output());
1527
const JSAtomState& names = gen->runtime->names();
1528
Label true_, done;
1529
1530
masm.branchTest32(Assembler::NonZero, input, input, &true_);
1531
masm.movePtr(ImmGCPtr(names.false_), output);
1532
masm.jump(&done);
1533
1534
masm.bind(&true_);
1535
masm.movePtr(ImmGCPtr(names.true_), output);
1536
1537
masm.bind(&done);
1538
}
1539
1540
void CodeGenerator::emitIntToString(Register input, Register output,
1541
Label* ool) {
1542
masm.boundsCheck32PowerOfTwo(input, StaticStrings::INT_STATIC_LIMIT, ool);
1543
1544
// Fast path for small integers.
1545
masm.movePtr(ImmPtr(&gen->runtime->staticStrings().intStaticTable), output);
1546
masm.loadPtr(BaseIndex(output, input, ScalePointer), output);
1547
}
1548
1549
void CodeGenerator::visitIntToString(LIntToString* lir) {
1550
Register input = ToRegister(lir->input());
1551
Register output = ToRegister(lir->output());
1552
1553
using Fn = JSFlatString* (*)(JSContext*, int);
1554
OutOfLineCode* ool = oolCallVM<Fn, Int32ToString<CanGC>>(
1555
lir, ArgList(input), StoreRegisterTo(output));
1556
1557
emitIntToString(input, output, ool->entry());
1558
1559
masm.bind(ool->rejoin());
1560
}
1561
1562
void CodeGenerator::visitDoubleToString(LDoubleToString* lir) {
1563
FloatRegister input = ToFloatRegister(lir->input());
1564
Register temp = ToRegister(lir->tempInt());
1565
Register output = ToRegister(lir->output());
1566
1567
using Fn = JSString* (*)(JSContext*, double);
1568
OutOfLineCode* ool = oolCallVM<Fn, NumberToString<CanGC>>(
1569
lir, ArgList(input), StoreRegisterTo(output));
1570
1571
// Try double to integer conversion and run integer to string code.
1572
masm.convertDoubleToInt32(input, temp, ool->entry(), false);
1573
emitIntToString(temp, output, ool->entry());
1574
1575
masm.bind(ool->rejoin());
1576
}
1577
1578
void CodeGenerator::visitValueToString(LValueToString* lir) {
1579
ValueOperand input = ToValue(lir, LValueToString::Input);
1580
Register output = ToRegister(lir->output());
1581
1582
using Fn = JSString* (*)(JSContext*, HandleValue);
1583
OutOfLineCode* ool = oolCallVM<Fn, ToStringSlow<CanGC>>(
1584
lir, ArgList(input), StoreRegisterTo(output));
1585
1586
Label done;
1587
Register tag = masm.extractTag(input, output);
1588
const JSAtomState& names = gen->runtime->names();
1589
1590
// String
1591
if (lir->mir()->input()->mightBeType(MIRType::String)) {
1592
Label notString;
1593
masm.branchTestString(Assembler::NotEqual, tag, &notString);
1594
masm.unboxString(input, output);
1595
masm.jump(&done);
1596
masm.bind(&notString);
1597
}