path: root/js/src/jit/Ion.cpp

/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
 * vim: set ts=8 sts=4 et sw=4 tw=99:
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#include "jit/Ion.h"

#include "mozilla/IntegerPrintfMacros.h"
#include "mozilla/MemoryReporting.h"
#include "mozilla/SizePrintfMacros.h"
#include "mozilla/ThreadLocal.h"

#include "jscompartment.h"
#include "jsgc.h"
#include "jsprf.h"

#include "gc/Marking.h"
#include "jit/AliasAnalysis.h"
#include "jit/AlignmentMaskAnalysis.h"
#include "jit/BacktrackingAllocator.h"
#include "jit/BaselineFrame.h"
#include "jit/BaselineInspector.h"
#include "jit/BaselineJIT.h"
#include "jit/CodeGenerator.h"
#include "jit/EagerSimdUnbox.h"
#include "jit/EdgeCaseAnalysis.h"
#include "jit/EffectiveAddressAnalysis.h"
#include "jit/FlowAliasAnalysis.h"
#include "jit/FoldLinearArithConstants.h"
#include "jit/InstructionReordering.h"
#include "jit/IonAnalysis.h"
#include "jit/IonBuilder.h"
#include "jit/IonOptimizationLevels.h"
#include "jit/JitcodeMap.h"
#include "jit/JitCommon.h"
#include "jit/JitCompartment.h"
#include "jit/JitSpewer.h"
#include "jit/LICM.h"
#include "jit/LIR.h"
#include "jit/LoopUnroller.h"
#include "jit/Lowering.h"
#include "jit/PerfSpewer.h"
#include "jit/RangeAnalysis.h"
#include "jit/ScalarReplacement.h"
#include "jit/Sink.h"
#include "jit/StupidAllocator.h"
#include "jit/ValueNumbering.h"
#include "jit/WasmBCE.h"
#include "vm/Debugger.h"
#include "vm/HelperThreads.h"
#include "vm/TraceLogging.h"

#include "jscompartmentinlines.h"
#include "jsobjinlines.h"
#include "jsscriptinlines.h"

#include "jit/JitFrames-inl.h"
#include "jit/shared/Lowering-shared-inl.h"
#include "vm/Debugger-inl.h"
#include "vm/EnvironmentObject-inl.h"
#include "vm/Stack-inl.h"

using namespace js;
using namespace js::jit;

// Assert that JitCode is gc::Cell aligned.
JS_STATIC_ASSERT(sizeof(JitCode) % gc::CellSize == 0);

static MOZ_THREAD_LOCAL(JitContext*) TlsJitContext;

static JitContext*
CurrentJitContext()
{
    if (!TlsJitContext.init())
        return nullptr;
    return TlsJitContext.get();
}

void
jit::SetJitContext(JitContext* ctx)
{
    TlsJitContext.set(ctx);
}

JitContext*
jit::GetJitContext()
{
    MOZ_ASSERT(CurrentJitContext());
    return CurrentJitContext();
}

JitContext*
jit::MaybeGetJitContext()
{
    return CurrentJitContext();
}

JitContext::JitContext(JSContext* cx, TempAllocator* temp)
  : cx(cx),
    temp(temp),
    runtime(CompileRuntime::get(cx->runtime())),
    compartment(CompileCompartment::get(cx->compartment())),
    prev_(CurrentJitContext()),
    assemblerCount_(0)
{
    SetJitContext(this);
}

JitContext::JitContext(ExclusiveContext* cx, TempAllocator* temp)
  : cx(nullptr),
    temp(temp),
    runtime(CompileRuntime::get(cx->runtime_)),
    compartment(nullptr),
    prev_(CurrentJitContext()),
    assemblerCount_(0)
{
    SetJitContext(this);
}

JitContext::JitContext(CompileRuntime* rt, CompileCompartment* comp, TempAllocator* temp)
  : cx(nullptr),
    temp(temp),
    runtime(rt),
    compartment(comp),
    prev_(CurrentJitContext()),
    assemblerCount_(0)
{
    SetJitContext(this);
}

JitContext::JitContext(CompileRuntime* rt)
  : cx(nullptr),
    temp(nullptr),
    runtime(rt),
    compartment(nullptr),
    prev_(CurrentJitContext()),
    assemblerCount_(0)
{
    SetJitContext(this);
}

JitContext::JitContext(TempAllocator* temp)
  : cx(nullptr),
    temp(temp),
    runtime(nullptr),
    compartment(nullptr),
    prev_(CurrentJitContext()),
    assemblerCount_(0)
{
    SetJitContext(this);
}

JitContext::JitContext(CompileRuntime* rt, TempAllocator* temp)
  : cx(nullptr),
    temp(temp),
    runtime(rt),
    compartment(nullptr),
    prev_(CurrentJitContext()),
    assemblerCount_(0)
{
    SetJitContext(this);
}

JitContext::JitContext()
  : cx(nullptr),
    temp(nullptr),
    runtime(nullptr),
    compartment(nullptr),
    prev_(CurrentJitContext()),
    assemblerCount_(0)
{
    SetJitContext(this);
}

JitContext::~JitContext()
{
    SetJitContext(prev_);
}

bool
jit::InitializeIon()
{
    if (!TlsJitContext.init())
        return false;
    CheckLogging();
#if defined(JS_CODEGEN_ARM)
    InitARMFlags();
#endif
    CheckPerf();
    return true;
}

JitRuntime::JitRuntime(JSRuntime* rt)
  : execAlloc_(rt),
    backedgeExecAlloc_(rt),
    exceptionTail_(nullptr),
    bailoutTail_(nullptr),
    profilerExitFrameTail_(nullptr),
    enterJIT_(nullptr),
    bailoutHandler_(nullptr),
    argumentsRectifier_(nullptr),
    argumentsRectifierReturnAddr_(nullptr),
    invalidator_(nullptr),
    debugTrapHandler_(nullptr),
    baselineDebugModeOSRHandler_(nullptr),
    functionWrappers_(nullptr),
    osrTempData_(nullptr),
    preventBackedgePatching_(false),
    backedgeTarget_(BackedgeLoopHeader),
    ionReturnOverride_(MagicValue(JS_ARG_POISON)),
    jitcodeGlobalTable_(nullptr)
{
}

JitRuntime::~JitRuntime()
{
    js_delete(functionWrappers_);
    freeOsrTempData();

    // By this point, the jitcode global table should be empty.
    MOZ_ASSERT_IF(jitcodeGlobalTable_, jitcodeGlobalTable_->empty());
    js_delete(jitcodeGlobalTable_);
}

bool
JitRuntime::initialize(JSContext* cx, AutoLockForExclusiveAccess& lock)
{
    AutoCompartment ac(cx, cx->atomsCompartment(lock), &lock);

    JitContext jctx(cx, nullptr);

    if (!cx->compartment()->ensureJitCompartmentExists(cx))
        return false;

    functionWrappers_ = cx->new_<VMWrapperMap>(cx);
    if (!functionWrappers_ || !functionWrappers_->init())
        return false;

    JitSpew(JitSpew_Codegen, "# Emitting profiler exit frame tail stub");
    profilerExitFrameTail_ = generateProfilerExitFrameTailStub(cx);
    if (!profilerExitFrameTail_)
        return false;

    JitSpew(JitSpew_Codegen, "# Emitting exception tail stub");

    void* handler = JS_FUNC_TO_DATA_PTR(void*, jit::HandleException);

    exceptionTail_ = generateExceptionTailStub(cx, handler);
    if (!exceptionTail_)
        return false;

    JitSpew(JitSpew_Codegen, "# Emitting bailout tail stub");
    bailoutTail_ = generateBailoutTailStub(cx);
    if (!bailoutTail_)
        return false;

    if (cx->runtime()->jitSupportsFloatingPoint) {
        JitSpew(JitSpew_Codegen, "# Emitting bailout tables");

        // Initialize some Ion-only stubs that require floating-point support.
        if (!bailoutTables_.reserve(FrameSizeClass::ClassLimit().classId()))
            return false;

        for (uint32_t id = 0;; id++) {
            FrameSizeClass class_ = FrameSizeClass::FromClass(id);
            if (class_ == FrameSizeClass::ClassLimit())
                break;
            bailoutTables_.infallibleAppend((JitCode*)nullptr);
            JitSpew(JitSpew_Codegen, "# Bailout table");
            bailoutTables_[id] = generateBailoutTable(cx, id);
            if (!bailoutTables_[id])
                return false;
        }

        JitSpew(JitSpew_Codegen, "# Emitting bailout handler");
        bailoutHandler_ = generateBailoutHandler(cx);
        if (!bailoutHandler_)
            return false;

        JitSpew(JitSpew_Codegen, "# Emitting invalidator");
        invalidator_ = generateInvalidator(cx);
        if (!invalidator_)
            return false;
    }

    JitSpew(JitSpew_Codegen, "# Emitting sequential arguments rectifier");
    argumentsRectifier_ = generateArgumentsRectifier(cx, &argumentsRectifierReturnAddr_);
    if (!argumentsRectifier_)
        return false;

    JitSpew(JitSpew_Codegen, "# Emitting EnterJIT sequence");
    enterJIT_ = generateEnterJIT(cx, EnterJitOptimized);
    if (!enterJIT_)
        return false;

    JitSpew(JitSpew_Codegen, "# Emitting EnterBaselineJIT sequence");
    enterBaselineJIT_ = generateEnterJIT(cx, EnterJitBaseline);
    if (!enterBaselineJIT_)
        return false;

    JitSpew(JitSpew_Codegen, "# Emitting Pre Barrier for Value");
    valuePreBarrier_ = generatePreBarrier(cx, MIRType::Value);
    if (!valuePreBarrier_)
        return false;

    JitSpew(JitSpew_Codegen, "# Emitting Pre Barrier for String");
    stringPreBarrier_ = generatePreBarrier(cx, MIRType::String);
    if (!stringPreBarrier_)
        return false;

    JitSpew(JitSpew_Codegen, "# Emitting Pre Barrier for Object");
    objectPreBarrier_ = generatePreBarrier(cx, MIRType::Object);
    if (!objectPreBarrier_)
        return false;

    JitSpew(JitSpew_Codegen, "# Emitting Pre Barrier for Shape");
    shapePreBarrier_ = generatePreBarrier(cx, MIRType::Shape);
    if (!shapePreBarrier_)
        return false;

    JitSpew(JitSpew_Codegen, "# Emitting Pre Barrier for ObjectGroup");
    objectGroupPreBarrier_ = generatePreBarrier(cx, MIRType::ObjectGroup);
    if (!objectGroupPreBarrier_)
        return false;

    JitSpew(JitSpew_Codegen, "# Emitting malloc stub");
    mallocStub_ = generateMallocStub(cx);
    if (!mallocStub_)
        return false;

    JitSpew(JitSpew_Codegen, "# Emitting free stub");
    freeStub_ = generateFreeStub(cx);
    if (!freeStub_)
        return false;

    JitSpew(JitSpew_Codegen, "# Emitting VM function wrappers");
    for (VMFunction* fun = VMFunction::functions; fun; fun = fun->next) {
        JitSpew(JitSpew_Codegen, "# VM function wrapper");
        if (!generateVMWrapper(cx, *fun))
            return false;
    }

    JitSpew(JitSpew_Codegen, "# Emitting lazy link stub");
    lazyLinkStub_ = generateLazyLinkStub(cx);
    if (!lazyLinkStub_)
        return false;

    jitcodeGlobalTable_ = cx->new_<JitcodeGlobalTable>();
    if (!jitcodeGlobalTable_)
        return false;

    return true;
}

JitCode*
JitRuntime::debugTrapHandler(JSContext* cx)
{
    if (!debugTrapHandler_) {
        // JitRuntime code stubs are shared across compartments and have to
        // be allocated in the atoms compartment.
        AutoLockForExclusiveAccess lock(cx);
        AutoCompartment ac(cx, cx->runtime()->atomsCompartment(lock), &lock);
        debugTrapHandler_ = generateDebugTrapHandler(cx);
    }
    return debugTrapHandler_;
}

uint8_t*
JitRuntime::allocateOsrTempData(size_t size)
{
    osrTempData_ = (uint8_t*)js_realloc(osrTempData_, size);
    return osrTempData_;
}

void
JitRuntime::freeOsrTempData()
{
    js_free(osrTempData_);
    osrTempData_ = nullptr;
}

void
JitRuntime::patchIonBackedges(JSRuntime* rt, BackedgeTarget target)
{
    if (target == BackedgeLoopHeader) {
        // We must be on the main thread. The caller must use
        // AutoPreventBackedgePatching to ensure we don't reenter.
        MOZ_ASSERT(preventBackedgePatching_);
        MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
    } else {
        // We must be called from InterruptRunningJitCode, or a signal handler
        // triggered there. rt->handlingJitInterrupt() ensures we can't reenter
        // this code.
        MOZ_ASSERT(!preventBackedgePatching_);
        MOZ_ASSERT(rt->handlingJitInterrupt());
    }

    // Do nothing if we know all backedges are already jumping to `target`.
    if (backedgeTarget_ == target)
        return;

    backedgeTarget_ = target;

    backedgeExecAlloc_.makeAllWritable();

    // Patch all loop backedges in Ion code so that they either jump to the
    // normal loop header or to an interrupt handler each time they run.
    for (InlineListIterator<PatchableBackedge> iter(backedgeList_.begin());
         iter != backedgeList_.end();
         iter++)
    {
        PatchableBackedge* patchableBackedge = *iter;
        if (target == BackedgeLoopHeader)
            PatchBackedge(patchableBackedge->backedge, patchableBackedge->loopHeader, target);
        else
            PatchBackedge(patchableBackedge->backedge, patchableBackedge->interruptCheck, target);
    }

    backedgeExecAlloc_.makeAllExecutable();
}

JitCompartment::JitCompartment()
  : stubCodes_(nullptr),
    cacheIRStubCodes_(nullptr),
    baselineGetPropReturnAddr_(nullptr),
    baselineSetPropReturnAddr_(nullptr),
    stringConcatStub_(nullptr),
    regExpMatcherStub_(nullptr),
    regExpSearcherStub_(nullptr),
    regExpTesterStub_(nullptr)
{
    baselineCallReturnAddrs_[0] = baselineCallReturnAddrs_[1] = nullptr;
}

JitCompartment::~JitCompartment()
{
    js_delete(stubCodes_);
    js_delete(cacheIRStubCodes_);
}

bool
JitCompartment::initialize(JSContext* cx)
{
    stubCodes_ = cx->new_<ICStubCodeMap>(cx->runtime());
    if (!stubCodes_)
        return false;

    if (!stubCodes_->init()) {
        ReportOutOfMemory(cx);
        return false;
    }

    cacheIRStubCodes_ = cx->new_<CacheIRStubCodeMap>(cx->runtime());
    if (!cacheIRStubCodes_)
        return false;

    if (!cacheIRStubCodes_->init()) {
        ReportOutOfMemory(cx);
        return false;
    }

    return true;
}

bool
JitCompartment::ensureIonStubsExist(JSContext* cx)
{
    if (!stringConcatStub_) {
        stringConcatStub_ = generateStringConcatStub(cx);
        if (!stringConcatStub_)
            return false;
    }

    return true;
}

void
jit::FinishOffThreadBuilder(JSRuntime* runtime, IonBuilder* builder,
                            const AutoLockHelperThreadState& locked)
{
    // Clean the references to the pending IonBuilder, if we just finished it.
    if (builder->script()->baselineScript()->hasPendingIonBuilder() &&
        builder->script()->baselineScript()->pendingIonBuilder() == builder)
    {
        builder->script()->baselineScript()->removePendingIonBuilder(builder->script());
    }

    // If the builder is still in one of the helper thread list, then remove it.
    if (builder->isInList()) {
        MOZ_ASSERT(runtime);
        runtime->ionLazyLinkListRemove(builder);
    }

    // Clear the recompiling flag of the old ionScript, since we continue to
    // use the old ionScript if recompiling fails.
    if (builder->script()->hasIonScript())
        builder->script()->ionScript()->clearRecompiling();

    // Clean up if compilation did not succeed.
    if (builder->script()->isIonCompilingOffThread()) {
        IonScript* ion =
            builder->abortReason() == AbortReason_Disable ? ION_DISABLED_SCRIPT : nullptr;
        builder->script()->setIonScript(runtime, ion);
    }

    // The builder is allocated into its LifoAlloc, so destroying that will
    // destroy the builder and all other data accumulated during compilation,
    // except any final codegen (which includes an assembler and needs to be
    // explicitly destroyed).
    js_delete(builder->backgroundCodegen());
    js_delete(builder->alloc().lifoAlloc());
}

static bool
LinkCodeGen(JSContext* cx, IonBuilder* builder, CodeGenerator *codegen)
{
    RootedScript script(cx, builder->script());
    TraceLoggerThread* logger = TraceLoggerForMainThread(cx->runtime());
    TraceLoggerEvent event(logger, TraceLogger_AnnotateScripts, script);
    AutoTraceLog logScript(logger, event);
    AutoTraceLog logLink(logger, TraceLogger_IonLinking);

    if (!codegen->link(cx, builder->constraints()))
        return false;

    return true;
}

static bool
LinkBackgroundCodeGen(JSContext* cx, IonBuilder* builder)
{
    CodeGenerator* codegen = builder->backgroundCodegen();
    if (!codegen)
        return false;

    JitContext jctx(cx, &builder->alloc());

    // Root the assembler until the builder is finished below. As it was
    // constructed off thread, the assembler has not been rooted previously,
    // though any GC activity would discard the builder.
    MacroAssembler::AutoRooter masm(cx, &codegen->masm);

    return LinkCodeGen(cx, builder, codegen);
}

void
jit::LinkIonScript(JSContext* cx, HandleScript calleeScript)
{
    IonBuilder* builder;

    {
        AutoLockHelperThreadState lock;

        // Get the pending builder from the Ion frame.
        MOZ_ASSERT(calleeScript->hasBaselineScript());
        builder = calleeScript->baselineScript()->pendingIonBuilder();
        calleeScript->baselineScript()->removePendingIonBuilder(calleeScript);

        // Remove from pending.
        cx->runtime()->ionLazyLinkListRemove(builder);
    }

    {
        AutoEnterAnalysis enterTypes(cx);
        if (!LinkBackgroundCodeGen(cx, builder)) {
            // Silently ignore OOM during code generation. The assembly code
            // doesn't has code to handle it after linking happened. So it's
            // not OK to throw a catchable exception from there.
            cx->clearPendingException();
        }
    }

    {
        AutoLockHelperThreadState lock;
        FinishOffThreadBuilder(cx->runtime(), builder, lock);
    }
}

uint8_t*
jit::LazyLinkTopActivation(JSContext* cx)
{
    // First frame should be an exit frame.
    JitFrameIterator it(cx);
    LazyLinkExitFrameLayout* ll = it.exitFrame()->as<LazyLinkExitFrameLayout>();
    RootedScript calleeScript(cx, ScriptFromCalleeToken(ll->jsFrame()->calleeToken()));

    LinkIonScript(cx, calleeScript);

    MOZ_ASSERT(calleeScript->hasBaselineScript());
    MOZ_ASSERT(calleeScript->baselineOrIonRawPointer());

    return calleeScript->baselineOrIonRawPointer();
}

/* static */ void
JitRuntime::Mark(JSTracer* trc, AutoLockForExclusiveAccess& lock)
{
    MOZ_ASSERT(!trc->runtime()->isHeapMinorCollecting());

    // Shared stubs are allocated in the atoms compartment, so do not iterate
    // them after the atoms heap after it has been "finished."
    if (trc->runtime()->atomsAreFinished())
        return;

    Zone* zone = trc->runtime()->atomsCompartment(lock)->zone();
    for (auto i = zone->cellIter<JitCode>(); !i.done(); i.next()) {
        JitCode* code = i;
        TraceRoot(trc, &code, "wrapper");
    }
}

/* static */ void
JitRuntime::MarkJitcodeGlobalTableUnconditionally(JSTracer* trc)
{
    if (trc->runtime()->spsProfiler.enabled() &&
        trc->runtime()->hasJitRuntime() &&
        trc->runtime()->jitRuntime()->hasJitcodeGlobalTable())
    {
        trc->runtime()->jitRuntime()->getJitcodeGlobalTable()->markUnconditionally(trc);
    }
}

/* static */ bool
JitRuntime::MarkJitcodeGlobalTableIteratively(JSTracer* trc)
{
    if (trc->runtime()->hasJitRuntime() &&
        trc->runtime()->jitRuntime()->hasJitcodeGlobalTable())
    {
        return trc->runtime()->jitRuntime()->getJitcodeGlobalTable()->markIteratively(trc);
    }
    return false;
}

/* static */ void
JitRuntime::SweepJitcodeGlobalTable(JSRuntime* rt)
{
    if (rt->hasJitRuntime() && rt->jitRuntime()->hasJitcodeGlobalTable())
        rt->jitRuntime()->getJitcodeGlobalTable()->sweep(rt);
}

void
JitCompartment::mark(JSTracer* trc, JSCompartment* compartment)
{
    // Free temporary OSR buffer.
    trc->runtime()->jitRuntime()->freeOsrTempData();
}

void
JitCompartment::sweep(FreeOp* fop, JSCompartment* compartment)
{
    // Any outstanding compilations should have been cancelled by the GC.
    MOZ_ASSERT(!HasOffThreadIonCompile(compartment));

    stubCodes_->sweep();
    cacheIRStubCodes_->sweep();

    // If the sweep removed the ICCall_Fallback stub, nullptr the baselineCallReturnAddr_ field.
    if (!stubCodes_->lookup(ICCall_Fallback::Compiler::BASELINE_CALL_KEY))
        baselineCallReturnAddrs_[0] = nullptr;
    if (!stubCodes_->lookup(ICCall_Fallback::Compiler::BASELINE_CONSTRUCT_KEY))
        baselineCallReturnAddrs_[1] = nullptr;

    // Similarly for the ICGetProp_Fallback stub.
    if (!stubCodes_->lookup(ICGetProp_Fallback::Compiler::BASELINE_KEY))
        baselineGetPropReturnAddr_ = nullptr;
    if (!stubCodes_->lookup(ICSetProp_Fallback::Compiler::BASELINE_KEY))
        baselineSetPropReturnAddr_ = nullptr;

    JSRuntime* rt = fop->runtime();
    if (stringConcatStub_ && !IsMarkedUnbarriered(rt, &stringConcatStub_))
        stringConcatStub_ = nullptr;

    if (regExpMatcherStub_ && !IsMarkedUnbarriered(rt, &regExpMatcherStub_))
        regExpMatcherStub_ = nullptr;

    if (regExpSearcherStub_ && !IsMarkedUnbarriered(rt, &regExpSearcherStub_))
        regExpSearcherStub_ = nullptr;

    if (regExpTesterStub_ && !IsMarkedUnbarriered(rt, &regExpTesterStub_))
        regExpTesterStub_ = nullptr;

    for (ReadBarrieredObject& obj : simdTemplateObjects_) {
        if (obj && IsAboutToBeFinalized(&obj))
            obj.set(nullptr);
    }
}

void
JitCompartment::toggleBarriers(bool enabled)
{
    // Toggle barriers in compartment wide stubs that have patchable pre barriers.
    if (regExpMatcherStub_)
        regExpMatcherStub_->togglePreBarriers(enabled, Reprotect);
    if (regExpSearcherStub_)
        regExpSearcherStub_->togglePreBarriers(enabled, Reprotect);
    if (regExpTesterStub_)
        regExpTesterStub_->togglePreBarriers(enabled, Reprotect);

    // Toggle barriers in baseline IC stubs.
    for (ICStubCodeMap::Enum e(*stubCodes_); !e.empty(); e.popFront()) {
        JitCode* code = *e.front().value().unsafeGet();
        code->togglePreBarriers(enabled, Reprotect);
    }
    for (CacheIRStubCodeMap::Enum e(*cacheIRStubCodes_); !e.empty(); e.popFront()) {
        JitCode* code = *e.front().value().unsafeGet();
        code->togglePreBarriers(enabled, Reprotect);
    }
}

size_t
JitCompartment::sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const
{
    size_t n = mallocSizeOf(this);
    if (stubCodes_)
        n += stubCodes_->sizeOfIncludingThis(mallocSizeOf);
    if (cacheIRStubCodes_)
        n += cacheIRStubCodes_->sizeOfIncludingThis(mallocSizeOf);
    return n;
}

JitCode*
JitRuntime::getBailoutTable(const FrameSizeClass& frameClass) const
{
    MOZ_ASSERT(frameClass != FrameSizeClass::None());
    return bailoutTables_[frameClass.classId()];
}

JitCode*
JitRuntime::getVMWrapper(const VMFunction& f) const
{
    MOZ_ASSERT(functionWrappers_);
    MOZ_ASSERT(functionWrappers_->initialized());
    JitRuntime::VMWrapperMap::Ptr p = functionWrappers_->readonlyThreadsafeLookup(&f);
    MOZ_ASSERT(p);

    return p->value();
}

template <AllowGC allowGC>
JitCode*
JitCode::New(JSContext* cx, uint8_t* code, uint32_t bufferSize, uint32_t headerSize,
             ExecutablePool* pool, CodeKind kind)
{
    JitCode* codeObj = Allocate<JitCode, allowGC>(cx);
    if (!codeObj) {
        pool->release(headerSize + bufferSize, kind);
        return nullptr;
    }

    new (codeObj) JitCode(code, bufferSize, headerSize, pool, kind);
    return codeObj;
}

template
JitCode*
JitCode::New<CanGC>(JSContext* cx, uint8_t* code, uint32_t bufferSize, uint32_t headerSize,
                    ExecutablePool* pool, CodeKind kind);

template
JitCode*
JitCode::New<NoGC>(JSContext* cx, uint8_t* code, uint32_t bufferSize, uint32_t headerSize,
                   ExecutablePool* pool, CodeKind kind);

void
JitCode::copyFrom(MacroAssembler& masm)
{
    // Store the JitCode pointer right before the code buffer, so we can
    // recover the gcthing from relocation tables.
    *(JitCode**)(code_ - sizeof(JitCode*)) = this;
    insnSize_ = masm.instructionsSize();
    masm.executableCopy(code_);

    jumpRelocTableBytes_ = masm.jumpRelocationTableBytes();
    masm.copyJumpRelocationTable(code_ + jumpRelocTableOffset());

    dataRelocTableBytes_ = masm.dataRelocationTableBytes();
    masm.copyDataRelocationTable(code_ + dataRelocTableOffset());

    preBarrierTableBytes_ = masm.preBarrierTableBytes();
    masm.copyPreBarrierTable(code_ + preBarrierTableOffset());

    masm.processCodeLabels(code_);
}

void
JitCode::traceChildren(JSTracer* trc)
{
    // Note that we cannot mark invalidated scripts, since we've basically
    // corrupted the code stream by injecting bailouts.
    if (invalidated())
        return;

    if (jumpRelocTableBytes_) {
        uint8_t* start = code_ + jumpRelocTableOffset();
        CompactBufferReader reader(start, start + jumpRelocTableBytes_);
        MacroAssembler::TraceJumpRelocations(trc, this, reader);
    }
    if (dataRelocTableBytes_) {
        // If we're moving objects, we need writable JIT code.
        bool movingObjects = trc->runtime()->isHeapMinorCollecting() || zone()->isGCCompacting();
        MaybeAutoWritableJitCode awjc(this, movingObjects ? Reprotect : DontReprotect);

        uint8_t* start = code_ + dataRelocTableOffset();
        CompactBufferReader reader(start, start + dataRelocTableBytes_);
        MacroAssembler::TraceDataRelocations(trc, this, reader);
    }
}

void
JitCode::finalize(FreeOp* fop)
{
    // If this jitcode had a bytecode map, it must have already been removed.
#ifdef DEBUG
    JSRuntime* rt = fop->runtime();
    if (hasBytecodeMap_) {
        MOZ_ASSERT(rt->jitRuntime()->hasJitcodeGlobalTable());
        MOZ_ASSERT(!rt->jitRuntime()->getJitcodeGlobalTable()->lookup(raw()));
    }
#endif

    MOZ_ASSERT(pool_);

    // With W^X JIT code, reprotecting memory for each JitCode instance is
    // slow, so we record the ranges and poison them later all at once. It's
    // safe to ignore OOM here, it just means we won't poison the code.
    if (fop->appendJitPoisonRange(JitPoisonRange(pool_, code_ - headerSize_,
                                                 headerSize_ + bufferSize_)))
    {
        pool_->addRef();
    }
    code_ = nullptr;

    // Code buffers are stored inside ExecutablePools. Pools are refcounted.
    // Releasing the pool may free it. Horrible hack: if we are using perf
    // integration, we don't want to reuse code addresses, so we just leak the
    // memory instead.
    if (!PerfEnabled())
        pool_->release(headerSize_ + bufferSize_, CodeKind(kind_));
    pool_ = nullptr;
}

void
JitCode::togglePreBarriers(bool enabled, ReprotectCode reprotect)
{
    uint8_t* start = code_ + preBarrierTableOffset();
    CompactBufferReader reader(start, start + preBarrierTableBytes_);

    if (!reader.more())
        return;

    MaybeAutoWritableJitCode awjc(this, reprotect);
    do {
        size_t offset = reader.readUnsigned();
        CodeLocationLabel loc(this, CodeOffset(offset));
        if (enabled)
            Assembler::ToggleToCmp(loc);
        else
            Assembler::ToggleToJmp(loc);
    } while (reader.more());
}

IonScript::IonScript()
  : method_(nullptr),
    deoptTable_(nullptr),
    osrPc_(nullptr),
    osrEntryOffset_(0),
    skipArgCheckEntryOffset_(0),
    invalidateEpilogueOffset_(0),
    invalidateEpilogueDataOffset_(0),
    numBailouts_(0),
    hasProfilingInstrumentation_(false),
    recompiling_(false),
    runtimeData_(0),
    runtimeSize_(0),
    cacheIndex_(0),
    cacheEntries_(0),
    safepointIndexOffset_(0),
    safepointIndexEntries_(0),
    safepointsStart_(0),
    safepointsSize_(0),
    frameSlots_(0),
    frameSize_(0),
    bailoutTable_(0),
    bailoutEntries_(0),
    osiIndexOffset_(0),
    osiIndexEntries_(0),
    snapshots_(0),
    snapshotsListSize_(0),
    snapshotsRVATableSize_(0),
    constantTable_(0),
    constantEntries_(0),
    backedgeList_(0),
    backedgeEntries_(0),
    invalidationCount_(0),
    recompileInfo_(),
    osrPcMismatchCounter_(0),
    fallbackStubSpace_()
{
}

IonScript*
IonScript::New(JSContext* cx, RecompileInfo recompileInfo,
               uint32_t frameSlots, uint32_t argumentSlots, uint32_t frameSize,
               size_t snapshotsListSize, size_t snapshotsRVATableSize,
               size_t recoversSize, size_t bailoutEntries,
               size_t constants, size_t safepointIndices,
               size_t osiIndices, size_t cacheEntries,
               size_t runtimeSize,  size_t safepointsSize,
               size_t backedgeEntries, size_t sharedStubEntries,
               OptimizationLevel optimizationLevel)
{
    constexpr size_t DataAlignment = sizeof(void*);

    if (snapshotsListSize >= MAX_BUFFER_SIZE ||
        (bailoutEntries >= MAX_BUFFER_SIZE / sizeof(uint32_t)))
    {
        ReportOutOfMemory(cx);
        return nullptr;
    }

    // This should not overflow on x86, because the memory is already allocated
    // *somewhere* and if their total overflowed there would be no memory left
    // at all.
    size_t paddedSnapshotsSize = AlignBytes(snapshotsListSize + snapshotsRVATableSize, DataAlignment);
    size_t paddedRecoversSize = AlignBytes(recoversSize, DataAlignment);
    size_t paddedBailoutSize = AlignBytes(bailoutEntries * sizeof(uint32_t), DataAlignment);
    size_t paddedConstantsSize = AlignBytes(constants * sizeof(Value), DataAlignment);
    size_t paddedSafepointIndicesSize = AlignBytes(safepointIndices * sizeof(SafepointIndex), DataAlignment);
    size_t paddedOsiIndicesSize = AlignBytes(osiIndices * sizeof(OsiIndex), DataAlignment);
    size_t paddedCacheEntriesSize = AlignBytes(cacheEntries * sizeof(uint32_t), DataAlignment);
    size_t paddedRuntimeSize = AlignBytes(runtimeSize, DataAlignment);
    size_t paddedSafepointSize = AlignBytes(safepointsSize, DataAlignment);
    size_t paddedBackedgeSize = AlignBytes(backedgeEntries * sizeof(PatchableBackedge), DataAlignment);
    size_t paddedSharedStubSize = AlignBytes(sharedStubEntries * sizeof(IonICEntry), DataAlignment);

    size_t bytes = paddedSnapshotsSize +
                   paddedRecoversSize +
                   paddedBailoutSize +
                   paddedConstantsSize +
                   paddedSafepointIndicesSize +
                   paddedOsiIndicesSize +
                   paddedCacheEntriesSize +
                   paddedRuntimeSize +
                   paddedSafepointSize +
                   paddedBackedgeSize +
                   paddedSharedStubSize;
    IonScript* script = cx->zone()->pod_malloc_with_extra<IonScript, uint8_t>(bytes);
    if (!script)
        return nullptr;
    new (script) IonScript();

    uint32_t offsetCursor = sizeof(IonScript);

    script->runtimeData_ = offsetCursor;
    script->runtimeSize_ = runtimeSize;
    offsetCursor += paddedRuntimeSize;

    script->cacheIndex_ = offsetCursor;
    script->cacheEntries_ = cacheEntries;
    offsetCursor += paddedCacheEntriesSize;

    script->safepointIndexOffset_ = offsetCursor;
    script->safepointIndexEntries_ = safepointIndices;
    offsetCursor += paddedSafepointIndicesSize;

    script->safepointsStart_ = offsetCursor;
    script->safepointsSize_ = safepointsSize;
    offsetCursor += paddedSafepointSize;

    script->bailoutTable_ = offsetCursor;
    script->bailoutEntries_ = bailoutEntries;
    offsetCursor += paddedBailoutSize;

    script->osiIndexOffset_ = offsetCursor;
    script->osiIndexEntries_ = osiIndices;
    offsetCursor += paddedOsiIndicesSize;

    script->snapshots_ = offsetCursor;
    script->snapshotsListSize_ = snapshotsListSize;
    script->snapshotsRVATableSize_ = snapshotsRVATableSize;
    offsetCursor += paddedSnapshotsSize;

    script->recovers_ = offsetCursor;
    script->recoversSize_ = recoversSize;
    offsetCursor += paddedRecoversSize;

    script->constantTable_ = offsetCursor;
    script->constantEntries_ = constants;
    offsetCursor += paddedConstantsSize;

    script->backedgeList_ = offsetCursor;
    script->backedgeEntries_ = backedgeEntries;
    offsetCursor += paddedBackedgeSize;

    script->sharedStubList_ = offsetCursor;
    script->sharedStubEntries_ = sharedStubEntries;
    offsetCursor += paddedSharedStubSize;

    script->frameSlots_ = frameSlots;
    script->argumentSlots_ = argumentSlots;

    script->frameSize_ = frameSize;

    script->recompileInfo_ = recompileInfo;
    script->optimizationLevel_ = optimizationLevel;

    return script;
}

void
IonScript::adoptFallbackStubs(FallbackICStubSpace* stubSpace)

{
    fallbackStubSpace()->adoptFrom(stubSpace);
}

void
IonScript::trace(JSTracer* trc)
{
    if (method_)
        TraceEdge(trc, &method_, "method");

    if (deoptTable_)
        TraceEdge(trc, &deoptTable_, "deoptimizationTable");

    for (size_t i = 0; i < numConstants(); i++)
        TraceEdge(trc, &getConstant(i), "constant");

    // Mark all IC stub codes hanging off the IC stub entries.
    for (size_t i = 0; i < numSharedStubs(); i++) {
        IonICEntry& ent = sharedStubList()[i];
        ent.trace(trc);
    }

    // Trace caches so that the JSScript pointer can be updated if moved.
    for (size_t i = 0; i < numCaches(); i++)
        getCacheFromIndex(i).trace(trc);
}

/* static */ void
IonScript::writeBarrierPre(Zone* zone, IonScript* ionScript)
{
    if (zone->needsIncrementalBarrier())
        ionScript->trace(zone->barrierTracer());
}

void
IonScript::copySnapshots(const SnapshotWriter* writer)
{
    MOZ_ASSERT(writer->listSize() == snapshotsListSize_);
    memcpy((uint8_t*)this + snapshots_,
           writer->listBuffer(), snapshotsListSize_);

    MOZ_ASSERT(snapshotsRVATableSize_);
    MOZ_ASSERT(writer->RVATableSize() == snapshotsRVATableSize_);
    memcpy((uint8_t*)this + snapshots_ + snapshotsListSize_,
           writer->RVATableBuffer(), snapshotsRVATableSize_);
}

void
IonScript::copyRecovers(const RecoverWriter* writer)
{
    MOZ_ASSERT(writer->size() == recoversSize_);
    memcpy((uint8_t*)this + recovers_, writer->buffer(), recoversSize_);
}

void
IonScript::copySafepoints(const SafepointWriter* writer)
{
    MOZ_ASSERT(writer->size() == safepointsSize_);
    memcpy((uint8_t*)this + safepointsStart_, writer->buffer(), safepointsSize_);
}

void
IonScript::copyBailoutTable(const SnapshotOffset* table)
{
    memcpy(bailoutTable(), table, bailoutEntries_ * sizeof(uint32_t));
}

void
IonScript::copyConstants(const Value* vp)
{
    for (size_t i = 0; i < constantEntries_; i++)
        constants()[i].init(vp[i]);
}

void
IonScript::copyPatchableBackedges(JSContext* cx, JitCode* code,
                                  PatchableBackedgeInfo* backedges,
                                  MacroAssembler& masm)
{
    JitRuntime* jrt = cx->runtime()->jitRuntime();
    JitRuntime::AutoPreventBackedgePatching apbp(cx->runtime());

    for (size_t i = 0; i < backedgeEntries_; i++) {
        PatchableBackedgeInfo& info = backedges[i];
        PatchableBackedge* patchableBackedge = &backedgeList()[i];

        info.backedge.fixup(&masm);
        CodeLocationJump backedge(code, info.backedge);
        CodeLocationLabel loopHeader(code, CodeOffset(info.loopHeader->offset()));
        CodeLocationLabel interruptCheck(code, CodeOffset(info.interruptCheck->offset()));
        new(patchableBackedge) PatchableBackedge(backedge, loopHeader, interruptCheck);

        // Point the backedge to either of its possible targets, matching the
        // other backedges in the runtime.
        if (jrt->backedgeTarget() == JitRuntime::BackedgeInterruptCheck)
            PatchBackedge(backedge, interruptCheck, JitRuntime::BackedgeInterruptCheck);
        else
            PatchBackedge(backedge, loopHeader, JitRuntime::BackedgeLoopHeader);

        jrt->addPatchableBackedge(patchableBackedge);
    }
}

void
IonScript::copySafepointIndices(const SafepointIndex* si, MacroAssembler& masm)
{
    // Jumps in the caches reflect the offset of those jumps in the compiled
    // code, not the absolute positions of the jumps. Update according to the
    // final code address now.
    SafepointIndex* table = safepointIndices();
    memcpy(table, si, safepointIndexEntries_ * sizeof(SafepointIndex));
}

void
IonScript::copyOsiIndices(const OsiIndex* oi, MacroAssembler& masm)
{
    memcpy(osiIndices(), oi, osiIndexEntries_ * sizeof(OsiIndex));
}

void
IonScript::copyRuntimeData(const uint8_t* data)
{
    memcpy(runtimeData(), data, runtimeSize());
}

void
IonScript::copyCacheEntries(const uint32_t* caches, MacroAssembler& masm)
{
    memcpy(cacheIndex(), caches, numCaches() * sizeof(uint32_t));

    // Jumps in the caches reflect the offset of those jumps in the compiled
    // code, not the absolute positions of the jumps. Update according to the
    // final code address now.
    for (size_t i = 0; i < numCaches(); i++)
        getCacheFromIndex(i).updateBaseAddress(method_, masm);
}

const SafepointIndex*
IonScript::getSafepointIndex(uint32_t disp) const
{
    MOZ_ASSERT(safepointIndexEntries_ > 0);

    const SafepointIndex* table = safepointIndices();
    if (safepointIndexEntries_ == 1) {
        MOZ_ASSERT(disp == table[0].displacement());
        return &table[0];
    }

    size_t minEntry = 0;
    size_t maxEntry = safepointIndexEntries_ - 1;
    uint32_t min = table[minEntry].displacement();
    uint32_t max = table[maxEntry].displacement();

    // Raise if the element is not in the list.
    MOZ_ASSERT(min <= disp && disp <= max);

    // Approximate the location of the FrameInfo.
    size_t guess = (disp - min) * (maxEntry - minEntry) / (max - min) + minEntry;
    uint32_t guessDisp = table[guess].displacement();

    if (table[guess].displacement() == disp)
        return &table[guess];

    // Doing a linear scan from the guess should be more efficient in case of
    // small group which are equally distributed on the code.
    //
    // such as:  <...      ...    ...  ...  .   ...    ...>
    if (guessDisp > disp) {
        while (--guess >= minEntry) {
            guessDisp = table[guess].displacement();
            MOZ_ASSERT(guessDisp >= disp);
            if (guessDisp == disp)
                return &table[guess];
        }
    } else {
        while (++guess <= maxEntry) {
            guessDisp = table[guess].displacement();
            MOZ_ASSERT(guessDisp <= disp);
            if (guessDisp == disp)
                return &table[guess];
        }
    }

    MOZ_CRASH("displacement not found.");
}

const OsiIndex*
IonScript::getOsiIndex(uint32_t disp) const
{
    const OsiIndex* end = osiIndices() + osiIndexEntries_;
    for (const OsiIndex* it = osiIndices(); it != end; ++it) {
        if (it->returnPointDisplacement() == disp)
            return it;
    }

    MOZ_CRASH("Failed to find OSI point return address");
}

const OsiIndex*
IonScript::getOsiIndex(uint8_t* retAddr) const
{
    JitSpew(JitSpew_IonInvalidate, "IonScript %p has method %p raw %p", (void*) this, (void*)
            method(), method()->raw());

    MOZ_ASSERT(containsCodeAddress(retAddr));
    uint32_t disp = retAddr - method()->raw();
    return getOsiIndex(disp);
}

void
IonScript::Trace(JSTracer* trc, IonScript* script)
{
    if (script != ION_DISABLED_SCRIPT)
        script->trace(trc);
}

void
IonScript::Destroy(FreeOp* fop, IonScript* script)
{
    script->unlinkFromRuntime(fop);

    /*
     * When the script contains pointers to nursery things, the store buffer can
     * contain entries that point into the fallback stub space. Since we can
     * destroy scripts outside the context of a GC, this situation could result
     * in us trying to mark invalid store buffer entries.
     *
     * Defer freeing any allocated blocks until after the next minor GC.
     */
    script->fallbackStubSpace_.freeAllAfterMinorGC(fop->runtime());

    fop->delete_(script);
}

void
JS::DeletePolicy<js::jit::IonScript>::operator()(const js::jit::IonScript* script)
{
    IonScript::Destroy(rt_->defaultFreeOp(), const_cast<IonScript*>(script));
}

void
IonScript::toggleBarriers(bool enabled, ReprotectCode reprotect)
{
    method()->togglePreBarriers(enabled, reprotect);
}

void
IonScript::purgeOptimizedStubs(Zone* zone)
{
    for (size_t i = 0; i < numSharedStubs(); i++) {
        IonICEntry& entry = sharedStubList()[i];
        if (!entry.hasStub())
            continue;

        ICStub* lastStub = entry.firstStub();
        while (lastStub->next())
            lastStub = lastStub->next();

        if (lastStub->isFallback()) {
            // Unlink all stubs allocated in the optimized space.
            ICStub* stub = entry.firstStub();
            ICStub* prev = nullptr;

            while (stub->next()) {
                if (!stub->allocatedInFallbackSpace()) {
                    lastStub->toFallbackStub()->unlinkStub(zone, prev, stub);
                    stub = stub->next();
                    continue;
                }

                prev = stub;
                stub = stub->next();
            }

            lastStub->toFallbackStub()->setInvalid();

            if (lastStub->isMonitoredFallback()) {
                // Monitor stubs can't make calls, so are always in the
                // optimized stub space.
                ICTypeMonitor_Fallback* lastMonStub =
                    lastStub->toMonitoredFallbackStub()->fallbackMonitorStub();
                lastMonStub->resetMonitorStubChain(zone);
                lastMonStub->setInvalid();
            }
        } else if (lastStub->isTypeMonitor_Fallback()) {
            lastStub->toTypeMonitor_Fallback()->resetMonitorStubChain(zone);
            lastStub->toTypeMonitor_Fallback()->setInvalid();
        } else {
            MOZ_ASSERT(lastStub->isTableSwitch());
        }
    }

#ifdef DEBUG
    // All remaining stubs must be allocated in the fallback space.
    for (size_t i = 0; i < numSharedStubs(); i++) {
        IonICEntry& entry = sharedStubList()[i];
        if (!entry.hasStub())
            continue;

        ICStub* stub = entry.firstStub();
        while (stub->next()) {
            MOZ_ASSERT(stub->allocatedInFallbackSpace());
            stub = stub->next();
        }
    }
#endif
}

void
IonScript::purgeCaches()
{
    // Don't reset any ICs if we're invalidated, otherwise, repointing the
    // inline jump could overwrite an invalidation marker. These ICs can
    // no longer run, however, the IC slow paths may be active on the stack.
    // ICs therefore are required to check for invalidation before patching,
    // to ensure the same invariant.
    if (invalidated())
        return;

    if (numCaches() == 0)
        return;

    AutoWritableJitCode awjc(method());
    for (size_t i = 0; i < numCaches(); i++)
        getCacheFromIndex(i).reset(DontReprotect);
}

void
IonScript::unlinkFromRuntime(FreeOp* fop)
{
    // The writes to the executable buffer below may clobber backedge jumps, so
    // make sure that those backedges are unlinked from the runtime and not
    // reclobbered with garbage if an interrupt is requested.
    JitRuntime* jrt = fop->runtime()->jitRuntime();
    JitRuntime::AutoPreventBackedgePatching apbp(fop->runtime());
    for (size_t i = 0; i < backedgeEntries_; i++)
        jrt->removePatchableBackedge(&backedgeList()[i]);

    // Clear the list of backedges, so that this method is idempotent. It is
    // called during destruction, and may be additionally called when the
    // script is invalidated.
    backedgeEntries_ = 0;
}

void
jit::ToggleBarriers(JS::Zone* zone, bool needs)
{
    JSRuntime* rt = zone->runtimeFromMainThread();
    if (!rt->hasJitRuntime())
        return;

    for (auto script = zone->cellIter<JSScript>(); !script.done(); script.next()) {
        if (script->hasIonScript())
            script->ionScript()->toggleBarriers(needs);
        if (script->hasBaselineScript())
            script->baselineScript()->toggleBarriers(needs);
    }

    for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next()) {
        if (comp->jitCompartment())
            comp->jitCompartment()->toggleBarriers(needs);
    }
}

namespace js {
namespace jit {

static void
OptimizeSinCos(MIRGenerator *mir, MIRGraph &graph)
{
    // Now, we are looking for:
    // var y = sin(x);
    // var z = cos(x);
    // Graph before:
    // - 1 op
    // - 6 mathfunction op1 Sin
    // - 7 mathfunction op1 Cos
    // Graph will look like:
    // - 1 op
    // - 5 sincos op1
    // - 6 mathfunction sincos5 Sin
    // - 7 mathfunction sincos5 Cos
    for (MBasicBlockIterator block(graph.begin()); block != graph.end(); block++) {
        for (MInstructionIterator iter(block->begin()), end(block->end()); iter != end; ) {
            MInstruction *ins = *iter++;
            if (!ins->isMathFunction() || ins->isRecoveredOnBailout())
                continue;

            MMathFunction *insFunc = ins->toMathFunction();
            if (insFunc->function() != MMathFunction::Sin && insFunc->function() != MMathFunction::Cos)
                continue;

            // Check if sin/cos is already optimized.
            if (insFunc->getOperand(0)->type() == MIRType::SinCosDouble)
                continue;

            // insFunc is either a |sin(x)| or |cos(x)| instruction. The
            // following loop iterates over the uses of |x| to check if both
            // |sin(x)| and |cos(x)| instructions exist.
            bool hasSin = false;
            bool hasCos = false;
            for (MUseDefIterator uses(insFunc->input()); uses; uses++)
            {
                if (!uses.def()->isInstruction())
                    continue;

                // We should replacing the argument of the sin/cos just when it
                // is dominated by the |block|.
                if (!block->dominates(uses.def()->block()))
                    continue;

                MInstruction *insUse = uses.def()->toInstruction();
                if (!insUse->isMathFunction() || insUse->isRecoveredOnBailout())
                    continue;

                MMathFunction *mathIns = insUse->toMathFunction();
                if (!hasSin && mathIns->function() == MMathFunction::Sin) {
                    hasSin = true;
                    JitSpew(JitSpew_Sincos, "Found sin in block %d.", mathIns->block()->id());
                }
                else if (!hasCos && mathIns->function() == MMathFunction::Cos) {
                    hasCos = true;
                    JitSpew(JitSpew_Sincos, "Found cos in block %d.", mathIns->block()->id());
                }

                if (hasCos && hasSin)
                    break;
            }

            if (!hasCos || !hasSin) {
                JitSpew(JitSpew_Sincos, "No sin/cos pair found.");
                continue;
            }

            JitSpew(JitSpew_Sincos, "Found, at least, a pair sin/cos. Adding sincos in block %d",
                    block->id());
            // Adding the MSinCos and replacing the parameters of the
            // sin(x)/cos(x) to sin(sincos(x))/cos(sincos(x)).
            MSinCos *insSinCos = MSinCos::New(graph.alloc(),
                                              insFunc->input(),
                                              insFunc->toMathFunction()->cache());
            insSinCos->setImplicitlyUsedUnchecked();
            block->insertBefore(insFunc, insSinCos);
            for (MUseDefIterator uses(insFunc->input()); uses; )
            {
                MDefinition* def = uses.def();
                uses++;
                if (!def->isInstruction())
                    continue;

                // We should replacing the argument of the sin/cos just when it
                // is dominated by the |block|.
                if (!block->dominates(def->block()))
                    continue;

                MInstruction *insUse = def->toInstruction();
                if (!insUse->isMathFunction() || insUse->isRecoveredOnBailout())
                    continue;

                MMathFunction *mathIns = insUse->toMathFunction();
                if (mathIns->function() != MMathFunction::Sin && mathIns->function() != MMathFunction::Cos)
                    continue;

                mathIns->replaceOperand(0, insSinCos);
                JitSpew(JitSpew_Sincos, "Replacing %s by sincos in block %d",
                        mathIns->function() == MMathFunction::Sin ? "sin" : "cos",
                        mathIns->block()->id());
            }
        }
    }
}

bool
OptimizeMIR(MIRGenerator* mir)
{
    MIRGraph& graph = mir->graph();
    GraphSpewer& gs = mir->graphSpewer();
    TraceLoggerThread* logger;
    if (GetJitContext()->onMainThread())
        logger = TraceLoggerForMainThread(GetJitContext()->runtime);
    else
        logger = TraceLoggerForCurrentThread();

    if (mir->shouldCancel("Start"))
        return false;

    gs.spewPass("BuildSSA");
    AssertBasicGraphCoherency(graph);

    if (!JitOptions.disablePgo && !mir->compilingWasm()) {
        AutoTraceLog log(logger, TraceLogger_PruneUnusedBranches);
        if (!PruneUnusedBranches(mir, graph))
            return false;
        gs.spewPass("Prune Unused Branches");
        AssertBasicGraphCoherency(graph);

        if (mir->shouldCancel("Prune Unused Branches"))
            return false;
    }

    {
        AutoTraceLog log(logger, TraceLogger_FoldTests);
        if (!FoldTests(graph))
            return false;
        gs.spewPass("Fold Tests");
        AssertBasicGraphCoherency(graph);

        if (mir->shouldCancel("Fold Tests"))
            return false;
    }

    {
        AutoTraceLog log(logger, TraceLogger_SplitCriticalEdges);
        if (!SplitCriticalEdges(graph))
            return false;
        gs.spewPass("Split Critical Edges");
        AssertGraphCoherency(graph);

        if (mir->shouldCancel("Split Critical Edges"))
            return false;
    }

    {
        AutoTraceLog log(logger, TraceLogger_RenumberBlocks);
        RenumberBlocks(graph);
        gs.spewPass("Renumber Blocks");
        AssertGraphCoherency(graph);

        if (mir->shouldCancel("Renumber Blocks"))
            return false;
    }

    {
        AutoTraceLog log(logger, TraceLogger_DominatorTree);
        if (!BuildDominatorTree(graph))
            return false;
        // No spew: graph not changed.

        if (mir->shouldCancel("Dominator Tree"))
            return false;
    }

    {
        AutoTraceLog log(logger, TraceLogger_PhiAnalysis);
        // Aggressive phi elimination must occur before any code elimination. If the
        // script contains a try-statement, we only compiled the try block and not
        // the catch or finally blocks, so in this case it's also invalid to use
        // aggressive phi elimination.
        Observability observability = graph.hasTryBlock()
                                      ? ConservativeObservability
                                      : AggressiveObservability;
        if (!EliminatePhis(mir, graph, observability))
            return false;
        gs.spewPass("Eliminate phis");
        AssertGraphCoherency(graph);

        if (mir->shouldCancel("Eliminate phis"))
            return false;

        if (!BuildPhiReverseMapping(graph))
            return false;
        AssertExtendedGraphCoherency(graph);
        // No spew: graph not changed.

        if (mir->shouldCancel("Phi reverse mapping"))
            return false;
    }

    if (!JitOptions.disableRecoverIns && mir->optimizationInfo().scalarReplacementEnabled()) {
        AutoTraceLog log(logger, TraceLogger_ScalarReplacement);
        if (!ScalarReplacement(mir, graph))
            return false;
        gs.spewPass("Scalar Replacement");
        AssertGraphCoherency(graph);

        if (mir->shouldCancel("Scalar Replacement"))
            return false;
    }

    if (!mir->compilingWasm()) {
        AutoTraceLog log(logger, TraceLogger_ApplyTypes);
        if (!ApplyTypeInformation(mir, graph))
            return false;
        gs.spewPass("Apply types");
        AssertExtendedGraphCoherency(graph);

        if (mir->shouldCancel("Apply types"))
            return false;
    }

    if (!JitOptions.disableRecoverIns && mir->optimizationInfo().eagerSimdUnboxEnabled()) {
        AutoTraceLog log(logger, TraceLogger_EagerSimdUnbox);
        if (!EagerSimdUnbox(mir, graph))
            return false;
        gs.spewPass("Eager Simd Unbox");
        AssertGraphCoherency(graph);

        if (mir->shouldCancel("Eager Simd Unbox"))
            return false;
    }

    if (mir->optimizationInfo().amaEnabled()) {
        AutoTraceLog log(logger, TraceLogger_AlignmentMaskAnalysis);
        AlignmentMaskAnalysis ama(graph);
        if (!ama.analyze())
            return false;
        gs.spewPass("Alignment Mask Analysis");
        AssertExtendedGraphCoherency(graph);

        if (mir->shouldCancel("Alignment Mask Analysis"))
            return false;
    }

    ValueNumberer gvn(mir, graph);
    if (!gvn.init())
        return false;

    // Alias analysis is required for LICM and GVN so that we don't move
    // loads across stores.
    if (mir->optimizationInfo().licmEnabled() ||
        mir->optimizationInfo().gvnEnabled())
    {
        {
            AutoTraceLog log(logger, TraceLogger_AliasAnalysis);
            if (JitOptions.disableFlowAA) {
                AliasAnalysis analysis(mir, graph);
                if (!analysis.analyze())
                    return false;
            } else {
                FlowAliasAnalysis analysis(mir, graph);
                if (!analysis.analyze())
                    return false;
            }

            gs.spewPass("Alias analysis");
            AssertExtendedGraphCoherency(graph);

            if (mir->shouldCancel("Alias analysis"))
                return false;
        }

        if (!mir->compilingWasm()) {
            // Eliminating dead resume point operands requires basic block
            // instructions to be numbered. Reuse the numbering computed during
            // alias analysis.
            if (!EliminateDeadResumePointOperands(mir, graph))
                return false;

            if (mir->shouldCancel("Eliminate dead resume point operands"))
                return false;
        }
    }

    if (mir->optimizationInfo().gvnEnabled()) {
        AutoTraceLog log(logger, TraceLogger_GVN);
        if (!gvn.run(ValueNumberer::UpdateAliasAnalysis))
            return false;
        gs.spewPass("GVN");
        AssertExtendedGraphCoherency(graph);

        if (mir->shouldCancel("GVN"))
            return false;
    }

    if (mir->optimizationInfo().licmEnabled()) {
        AutoTraceLog log(logger, TraceLogger_LICM);
        // LICM can hoist instructions from conditional branches and trigger
        // repeated bailouts. Disable it if this script is known to bailout
        // frequently.
        JSScript* script = mir->info().script();
        if (!script || !script->hadFrequentBailouts()) {
            if (!LICM(mir, graph))
                return false;
            gs.spewPass("LICM");
            AssertExtendedGraphCoherency(graph);

            if (mir->shouldCancel("LICM"))
                return false;
        }
    }

    RangeAnalysis r(mir, graph);
    if (mir->optimizationInfo().rangeAnalysisEnabled()) {
        AutoTraceLog log(logger, TraceLogger_RangeAnalysis);
        if (!r.addBetaNodes())
            return false;
        gs.spewPass("Beta");
        AssertExtendedGraphCoherency(graph);

        if (mir->shouldCancel("RA Beta"))
            return false;

        if (!r.analyze() || !r.addRangeAssertions())
            return false;
        gs.spewPass("Range Analysis");
        AssertExtendedGraphCoherency(graph);

        if (mir->shouldCancel("Range Analysis"))
            return false;

        if (!r.removeBetaNodes())
            return false;
        gs.spewPass("De-Beta");
        AssertExtendedGraphCoherency(graph);

        if (mir->shouldCancel("RA De-Beta"))
            return false;

        if (mir->optimizationInfo().gvnEnabled()) {
            bool shouldRunUCE = false;
            if (!r.prepareForUCE(&shouldRunUCE))
                return false;
            gs.spewPass("RA check UCE");
            AssertExtendedGraphCoherency(graph);

            if (mir->shouldCancel("RA check UCE"))
                return false;

            if (shouldRunUCE) {
                if (!gvn.run(ValueNumberer::DontUpdateAliasAnalysis))
                    return false;
                gs.spewPass("UCE After RA");
                AssertExtendedGraphCoherency(graph);

                if (mir->shouldCancel("UCE After RA"))
                    return false;
            }
        }

        if (mir->optimizationInfo().autoTruncateEnabled()) {
            if (!r.truncate())
                return false;
            gs.spewPass("Truncate Doubles");
            AssertExtendedGraphCoherency(graph);

            if (mir->shouldCancel("Truncate Doubles"))
                return false;
        }

        if (mir->optimizationInfo().loopUnrollingEnabled()) {
            AutoTraceLog log(logger, TraceLogger_LoopUnrolling);

            if (!UnrollLoops(graph, r.loopIterationBounds))
                return false;

            gs.spewPass("Unroll Loops");
            AssertExtendedGraphCoherency(graph);
        }
    }

    if (!JitOptions.disableRecoverIns) {
        AutoTraceLog log(logger, TraceLogger_Sink);
        if (!Sink(mir, graph))
            return false;
        gs.spewPass("Sink");
        AssertExtendedGraphCoherency(graph);

        if (mir->shouldCancel("Sink"))
            return false;
    }

    if (!JitOptions.disableRecoverIns && mir->optimizationInfo().rangeAnalysisEnabled()) {
        AutoTraceLog log(logger, TraceLogger_RemoveUnnecessaryBitops);
        if (!r.removeUnnecessaryBitops())
            return false;
        gs.spewPass("Remove Unnecessary Bitops");
        AssertExtendedGraphCoherency(graph);

        if (mir->shouldCancel("Remove Unnecessary Bitops"))
            return false;
    }

    {
        AutoTraceLog log(logger, TraceLogger_FoldLinearArithConstants);
        if (!FoldLinearArithConstants(mir, graph))
            return false;
        gs.spewPass("Fold Linear Arithmetic Constants");
        AssertBasicGraphCoherency(graph);

        if (mir->shouldCancel("Fold Linear Arithmetic Constants"))
            return false;
    }

    if (mir->optimizationInfo().eaaEnabled()) {
        AutoTraceLog log(logger, TraceLogger_EffectiveAddressAnalysis);
        EffectiveAddressAnalysis eaa(mir, graph);
        if (!eaa.analyze())
            return false;
        gs.spewPass("Effective Address Analysis");
        AssertExtendedGraphCoherency(graph);

        if (mir->shouldCancel("Effective Address Analysis"))
            return false;
    }

    if (mir->optimizationInfo().sincosEnabled()) {
        AutoTraceLog log(logger, TraceLogger_Sincos);
        OptimizeSinCos(mir, graph);
        gs.spewPass("Sincos optimization");
        AssertExtendedGraphCoherency(graph);

        if (mir->shouldCancel("Sincos optimization"))
            return false;
    }

    {
        AutoTraceLog log(logger, TraceLogger_EliminateDeadCode);
        if (!EliminateDeadCode(mir, graph))
            return false;
        gs.spewPass("DCE");
        AssertExtendedGraphCoherency(graph);

        if (mir->shouldCancel("DCE"))
            return false;
    }

    if (mir->optimizationInfo().instructionReorderingEnabled()) {
        AutoTraceLog log(logger, TraceLogger_ReorderInstructions);
        if (!ReorderInstructions(mir, graph))
            return false;
        gs.spewPass("Reordering");

        AssertExtendedGraphCoherency(graph);

        if (mir->shouldCancel("Reordering"))
            return false;
    }

    // Make loops contiguous. We do this after GVN/UCE and range analysis,
    // which can remove CFG edges, exposing more blocks that can be moved.
    {
        AutoTraceLog log(logger, TraceLogger_MakeLoopsContiguous);
        if (!MakeLoopsContiguous(graph))
            return false;
        gs.spewPass("Make loops contiguous");
        AssertExtendedGraphCoherency(graph);

        if (mir->shouldCancel("Make loops contiguous"))
            return false;
    }

    // Passes after this point must not move instructions; these analyses
    // depend on knowing the final order in which instructions will execute.

    if (mir->optimizationInfo().edgeCaseAnalysisEnabled()) {
        AutoTraceLog log(logger, TraceLogger_EdgeCaseAnalysis);
        EdgeCaseAnalysis edgeCaseAnalysis(mir, graph);
        if (!edgeCaseAnalysis.analyzeLate())
            return false;
        gs.spewPass("Edge Case Analysis (Late)");
        AssertGraphCoherency(graph);

        if (mir->shouldCancel("Edge Case Analysis (Late)"))
            return false;
    }

    if (mir->optimizationInfo().eliminateRedundantChecksEnabled()) {
        AutoTraceLog log(logger, TraceLogger_EliminateRedundantChecks);
        // Note: check elimination has to run after all other passes that move
        // instructions. Since check uses are replaced with the actual index,
        // code motion after this pass could incorrectly move a load or store
        // before its bounds check.
        if (!EliminateRedundantChecks(graph))
            return false;
        gs.spewPass("Bounds Check Elimination");
        AssertGraphCoherency(graph);
    }

    if (!mir->compilingWasm()) {
        AutoTraceLog log(logger, TraceLogger_AddKeepAliveInstructions);
        if (!AddKeepAliveInstructions(graph))
            return false;
        gs.spewPass("Add KeepAlive Instructions");
        AssertGraphCoherency(graph);
    }

    if (mir->compilingWasm()) {
        if (!EliminateBoundsChecks(mir, graph))
            return false;
        gs.spewPass("Redundant Bounds Check Elimination");
        AssertGraphCoherency(graph);
    }

    DumpMIRExpressions(graph);

    return true;
}

LIRGraph*
GenerateLIR(MIRGenerator* mir)
{
    MIRGraph& graph = mir->graph();
    GraphSpewer& gs = mir->graphSpewer();

    TraceLoggerThread* logger;
    if (GetJitContext()->onMainThread())
        logger = TraceLoggerForMainThread(GetJitContext()->runtime);
    else
        logger = TraceLoggerForCurrentThread();

    LIRGraph* lir = mir->alloc().lifoAlloc()->new_<LIRGraph>(&graph);
    if (!lir || !lir->init())
        return nullptr;

    LIRGenerator lirgen(mir, graph, *lir);
    {
        AutoTraceLog log(logger, TraceLogger_GenerateLIR);
        if (!lirgen.generate())
            return nullptr;
        gs.spewPass("Generate LIR");

        if (mir->shouldCancel("Generate LIR"))
            return nullptr;
    }

    AllocationIntegrityState integrity(*lir);

    {
        AutoTraceLog log(logger, TraceLogger_RegisterAllocation);

        IonRegisterAllocator allocator = mir->optimizationInfo().registerAllocator();

        switch (allocator) {
          case RegisterAllocator_Backtracking:
          case RegisterAllocator_Testbed: {
#ifdef DEBUG
            if (!integrity.record())
                return nullptr;
#endif

            BacktrackingAllocator regalloc(mir, &lirgen, *lir,
                                           allocator == RegisterAllocator_Testbed);
            if (!regalloc.go())
                return nullptr;

#ifdef DEBUG
            if (!integrity.check(false))
                return nullptr;
#endif

            gs.spewPass("Allocate Registers [Backtracking]");
            break;
          }

          case RegisterAllocator_Stupid: {
            // Use the integrity checker to populate safepoint information, so
            // run it in all builds.
            if (!integrity.record())
                return nullptr;

            StupidAllocator regalloc(mir, &lirgen, *lir);
            if (!regalloc.go())
                return nullptr;
            if (!integrity.check(true))
                return nullptr;
            gs.spewPass("Allocate Registers [Stupid]");
            break;
          }

          default:
            MOZ_CRASH("Bad regalloc");
        }

        if (mir->shouldCancel("Allocate Registers"))
            return nullptr;
    }

    return lir;
}

CodeGenerator*
GenerateCode(MIRGenerator* mir, LIRGraph* lir)
{
    TraceLoggerThread* logger;
    if (GetJitContext()->onMainThread())
        logger = TraceLoggerForMainThread(GetJitContext()->runtime);
    else
        logger = TraceLoggerForCurrentThread();
    AutoTraceLog log(logger, TraceLogger_GenerateCode);

    CodeGenerator* codegen = js_new<CodeGenerator>(mir, lir);
    if (!codegen)
        return nullptr;

    if (!codegen->generate()) {
        js_delete(codegen);
        return nullptr;
    }

    return codegen;
}

CodeGenerator*
CompileBackEnd(MIRGenerator* mir)
{
    // Everything in CompileBackEnd can potentially run on a helper thread.
    AutoEnterIonCompilation enter(mir->safeForMinorGC());
    AutoSpewEndFunction spewEndFunction(mir);

    if (!OptimizeMIR(mir))
        return nullptr;

    LIRGraph* lir = GenerateLIR(mir);
    if (!lir)
        return nullptr;

    return GenerateCode(mir, lir);
}

// Find a finished builder for the compartment.
static IonBuilder*
GetFinishedBuilder(JSContext* cx, GlobalHelperThreadState::IonBuilderVector& finished)
{
    for (size_t i = 0; i < finished.length(); i++) {
        IonBuilder* testBuilder = finished[i];
        if (testBuilder->compartment == CompileCompartment::get(cx->compartment())) {
            HelperThreadState().remove(finished, &i);
            return testBuilder;
        }
    }

    return nullptr;
}

void
AttachFinishedCompilations(JSContext* cx)
{
    JitCompartment* ion = cx->compartment()->jitCompartment();
    if (!ion)
        return;

    {
        AutoLockHelperThreadState lock;

        GlobalHelperThreadState::IonBuilderVector& finished = HelperThreadState().ionFinishedList(lock);

        // Incorporate any off thread compilations for the compartment which have
        // finished, failed or have been cancelled.
        while (true) {
            // Find a finished builder for the compartment.
            IonBuilder* builder = GetFinishedBuilder(cx, finished);
            if (!builder)
                break;

            JSScript* script = builder->script();
            MOZ_ASSERT(script->hasBaselineScript());
            script->baselineScript()->setPendingIonBuilder(cx->runtime(), script, builder);
            cx->runtime()->ionLazyLinkListAdd(builder);

            // Don't keep more than 100 lazy link builders.
            // Link the oldest ones immediately.
            while (cx->runtime()->ionLazyLinkListSize() > 100) {
                jit::IonBuilder* builder = cx->runtime()->ionLazyLinkList().getLast();
                RootedScript script(cx, builder->script());

                AutoUnlockHelperThreadState unlock(lock);
                AutoCompartment ac(cx, script->compartment());
                jit::LinkIonScript(cx, script);
            }

            continue;
        }
    }
}

static void
TrackAllProperties(JSContext* cx, JSObject* obj)
{
    MOZ_ASSERT(obj->isSingleton());

    for (Shape::Range<NoGC> range(obj->as<NativeObject>().lastProperty()); !range.empty(); range.popFront())
        EnsureTrackPropertyTypes(cx, obj, range.front().propid());
}

static void
TrackPropertiesForSingletonScopes(JSContext* cx, JSScript* script, BaselineFrame* baselineFrame)
{
    // Ensure that all properties of singleton call objects which the script
    // could access are tracked. These are generally accessed through
    // ALIASEDVAR operations in baseline and will not be tracked even if they
    // have been accessed in baseline code.
    JSObject* environment = script->functionNonDelazifying()
                            ? script->functionNonDelazifying()->environment()
                            : nullptr;

    while (environment && !environment->is<GlobalObject>()) {
        if (environment->is<CallObject>() && environment->isSingleton())
            TrackAllProperties(cx, environment);
        environment = environment->enclosingEnvironment();
    }

    if (baselineFrame) {
        JSObject* scope = baselineFrame->environmentChain();
        if (scope->is<CallObject>() && scope->isSingleton())
            TrackAllProperties(cx, scope);
    }
}

static void
TrackIonAbort(JSContext* cx, JSScript* script, jsbytecode* pc, const char* message)
{
    if (!cx->runtime()->jitRuntime()->isOptimizationTrackingEnabled(cx->runtime()))
        return;

    // Only bother tracking aborts of functions we're attempting to
    // Ion-compile after successfully running in Baseline.
    if (!script->hasBaselineScript())
        return;

    JitcodeGlobalTable* table = cx->runtime()->jitRuntime()->getJitcodeGlobalTable();
    void* ptr = script->baselineScript()->method()->raw();
    JitcodeGlobalEntry& entry = table->lookupInfallible(ptr);
    entry.baselineEntry().trackIonAbort(pc, message);
}

static void
TrackAndSpewIonAbort(JSContext* cx, JSScript* script, const char* message)
{
    JitSpew(JitSpew_IonAbort, "%s", message);
    TrackIonAbort(cx, script, script->code(), message);
}

static AbortReason
IonCompile(JSContext* cx, JSScript* script,
           BaselineFrame* baselineFrame, jsbytecode* osrPc,
           bool recompile, OptimizationLevel optimizationLevel)
{
    TraceLoggerThread* logger = TraceLoggerForMainThread(cx->runtime());
    TraceLoggerEvent event(logger, TraceLogger_AnnotateScripts, script);
    AutoTraceLog logScript(logger, event);
    AutoTraceLog logCompile(logger, TraceLogger_IonCompilation);

    // Make sure the script's canonical function isn't lazy. We can't de-lazify
    // it in a helper thread.
    script->ensureNonLazyCanonicalFunction();

    TrackPropertiesForSingletonScopes(cx, script, baselineFrame);

    LifoAlloc* alloc = cx->new_<LifoAlloc>(TempAllocator::PreferredLifoChunkSize);
    if (!alloc)
        return AbortReason_Alloc;

    ScopedJSDeletePtr<LifoAlloc> autoDelete(alloc);

    TempAllocator* temp = alloc->new_<TempAllocator>(alloc);
    if (!temp)
        return AbortReason_Alloc;

    JitContext jctx(cx, temp);

    if (!cx->compartment()->ensureJitCompartmentExists(cx))
        return AbortReason_Alloc;

    if (!cx->compartment()->jitCompartment()->ensureIonStubsExist(cx))
        return AbortReason_Alloc;

    MIRGraph* graph = alloc->new_<MIRGraph>(temp);
    if (!graph)
        return AbortReason_Alloc;

    InlineScriptTree* inlineScriptTree = InlineScriptTree::New(temp, nullptr, nullptr, script);
    if (!inlineScriptTree)
        return AbortReason_Alloc;

    CompileInfo* info = alloc->new_<CompileInfo>(script, script->functionNonDelazifying(), osrPc,
                                                 Analysis_None,
                                                 script->needsArgsObj(), inlineScriptTree);
    if (!info)
        return AbortReason_Alloc;

    BaselineInspector* inspector = alloc->new_<BaselineInspector>(script);
    if (!inspector)
        return AbortReason_Alloc;

    BaselineFrameInspector* baselineFrameInspector = nullptr;
    if (baselineFrame) {
        baselineFrameInspector = NewBaselineFrameInspector(temp, baselineFrame, info);
        if (!baselineFrameInspector)
            return AbortReason_Alloc;
    }

    CompilerConstraintList* constraints = NewCompilerConstraintList(*temp);
    if (!constraints)
        return AbortReason_Alloc;

    const OptimizationInfo* optimizationInfo = IonOptimizations.get(optimizationLevel);
    const JitCompileOptions options(cx);

    IonBuilder* builder = alloc->new_<IonBuilder>((JSContext*) nullptr,
                                                  CompileCompartment::get(cx->compartment()),
                                                  options, temp, graph, constraints,
                                                  inspector, info, optimizationInfo,
                                                  baselineFrameInspector);
    if (!builder)
        return AbortReason_Alloc;

    if (cx->runtime()->gc.storeBuffer.cancelIonCompilations())
        builder->setNotSafeForMinorGC();

    MOZ_ASSERT(recompile == builder->script()->hasIonScript());
    MOZ_ASSERT(builder->script()->canIonCompile());

    RootedScript builderScript(cx, builder->script());

    if (recompile)
        builderScript->ionScript()->setRecompiling();

    SpewBeginFunction(builder, builderScript);

    bool succeeded;
    {
        AutoEnterAnalysis enter(cx);
        succeeded = builder->build();
        builder->clearForBackEnd();
    }

    if (!succeeded) {
        AbortReason reason = builder->abortReason();
        builder->graphSpewer().endFunction();
        if (reason == AbortReason_PreliminaryObjects) {
            // Some group was accessed which has associated preliminary objects
            // to analyze. Do this now and we will try to build again shortly.
            const MIRGenerator::ObjectGroupVector& groups = builder->abortedPreliminaryGroups();
            for (size_t i = 0; i < groups.length(); i++) {
                ObjectGroup* group = groups[i];
                if (group->newScript()) {
                    if (!group->newScript()->maybeAnalyze(cx, group, nullptr, /* force = */ true))
                        return AbortReason_Alloc;
                } else if (group->maybePreliminaryObjects()) {
                    group->maybePreliminaryObjects()->maybeAnalyze(cx, group, /* force = */ true);
                } else {
                    MOZ_CRASH("Unexpected aborted preliminary group");
                }
            }
        }

        if (builder->hadActionableAbort()) {
            JSScript* abortScript;
            jsbytecode* abortPc;
            const char* abortMessage;
            builder->actionableAbortLocationAndMessage(&abortScript, &abortPc, &abortMessage);
            TrackIonAbort(cx, abortScript, abortPc, abortMessage);
        }

        if (cx->isThrowingOverRecursed()) {
            // Non-analysis compilations should never fail with stack overflow.
            MOZ_CRASH("Stack overflow during compilation");
        }

        return reason;
    }

    // If possible, compile the script off thread.
    if (options.offThreadCompilationAvailable()) {
        JitSpew(JitSpew_IonSyncLogs, "Can't log script %s:%" PRIuSIZE
                ". (Compiled on background thread.)",
                builderScript->filename(), builderScript->lineno());

        if (!CreateMIRRootList(*builder))
            return AbortReason_Alloc;

        if (!StartOffThreadIonCompile(cx, builder)) {
            JitSpew(JitSpew_IonAbort, "Unable to start off-thread ion compilation.");
            builder->graphSpewer().endFunction();
            return AbortReason_Alloc;
        }

        if (!recompile)
            builderScript->setIonScript(cx->runtime(), ION_COMPILING_SCRIPT);

        // The allocator and associated data will be destroyed after being
        // processed in the finishedOffThreadCompilations list.
        autoDelete.forget();

        return AbortReason_NoAbort;
    }

    {
        ScopedJSDeletePtr<CodeGenerator> codegen;
        AutoEnterAnalysis enter(cx);
        codegen = CompileBackEnd(builder);
        if (!codegen) {
            JitSpew(JitSpew_IonAbort, "Failed during back-end compilation.");
            if (cx->isExceptionPending())
                return AbortReason_Error;
            return AbortReason_Disable;
        }

        succeeded = LinkCodeGen(cx, builder, codegen);
    }

    if (succeeded)
        return AbortReason_NoAbort;
    if (cx->isExceptionPending())
        return AbortReason_Error;
    return AbortReason_Disable;
}

static bool
CheckFrame(JSContext* cx, BaselineFrame* frame)
{
    MOZ_ASSERT(!frame->script()->isStarGenerator());
    MOZ_ASSERT(!frame->script()->isLegacyGenerator());
    MOZ_ASSERT(!frame->script()->isAsync());
    MOZ_ASSERT(!frame->isDebuggerEvalFrame());
    MOZ_ASSERT(!frame->isEvalFrame());

    // This check is to not overrun the stack.
    if (frame->isFunctionFrame()) {
        if (TooManyActualArguments(frame->numActualArgs())) {
            TrackAndSpewIonAbort(cx, frame->script(), "too many actual arguments");
            return false;
        }

        if (TooManyFormalArguments(frame->numFormalArgs())) {
            TrackAndSpewIonAbort(cx, frame->script(), "too many arguments");
            return false;
        }
    }

    return true;
}

static bool
CheckScript(JSContext* cx, JSScript* script, bool osr)
{
    if (script->isForEval()) {
        // Eval frames are not yet supported. Supporting this will require new
        // logic in pushBailoutFrame to deal with linking prev.
        // Additionally, JSOP_DEFVAR support will require baking in isEvalFrame().
        TrackAndSpewIonAbort(cx, script, "eval script");
        return false;
    }

    if (script->isStarGenerator() || script->isLegacyGenerator()) {
        TrackAndSpewIonAbort(cx, script, "generator script");
        return false;
    }

    if (script->isAsync()) {
        TrackAndSpewIonAbort(cx, script, "async script");
        return false;
    }

    if (script->hasNonSyntacticScope() && !script->functionNonDelazifying()) {
        // Support functions with a non-syntactic global scope but not other
        // scripts. For global scripts, IonBuilder currently uses the global
        // object as scope chain, this is not valid when the script has a
        // non-syntactic global scope.
        TrackAndSpewIonAbort(cx, script, "has non-syntactic global scope");
        return false;
    }

    if (script->functionHasExtraBodyVarScope() &&
        script->functionExtraBodyVarScope()->hasEnvironment())
    {
        // This restriction will be lifted when intra-function scope chains
        // are compilable by Ion. See bug 1273858.
        TrackAndSpewIonAbort(cx, script, "has extra var environment");
        return false;
    }

    if (script->nTypeSets() >= UINT16_MAX) {
        // In this case multiple bytecode ops can share a single observed
        // TypeSet (see bug 1303710).
        TrackAndSpewIonAbort(cx, script, "too many typesets");
        return false;
    }

    return true;
}

static MethodStatus
CheckScriptSize(JSContext* cx, JSScript* script)
{
    if (!JitOptions.limitScriptSize)
        return Method_Compiled;

    uint32_t numLocalsAndArgs = NumLocalsAndArgs(script);

    if (script->length() > MAX_MAIN_THREAD_SCRIPT_SIZE ||
        numLocalsAndArgs > MAX_MAIN_THREAD_LOCALS_AND_ARGS)
    {
        if (!OffThreadCompilationAvailable(cx)) {
            JitSpew(JitSpew_IonAbort, "Script too large (%" PRIuSIZE " bytes) (%u locals/args)",
                    script->length(), numLocalsAndArgs);
            TrackIonAbort(cx, script, script->code(), "too large");
            return Method_CantCompile;
        }
    }

    return Method_Compiled;
}

bool
CanIonCompileScript(JSContext* cx, JSScript* script, bool osr)
{
    if (!script->canIonCompile() || !CheckScript(cx, script, osr))
        return false;

    return CheckScriptSize(cx, script) == Method_Compiled;
}

static OptimizationLevel
GetOptimizationLevel(HandleScript script, jsbytecode* pc)
{
    return IonOptimizations.levelForScript(script, pc);
}

static MethodStatus
Compile(JSContext* cx, HandleScript script, BaselineFrame* osrFrame, jsbytecode* osrPc,
        bool forceRecompile = false)
{
    MOZ_ASSERT(jit::IsIonEnabled(cx));
    MOZ_ASSERT(jit::IsBaselineEnabled(cx));
    MOZ_ASSERT_IF(osrPc != nullptr, LoopEntryCanIonOsr(osrPc));

    if (!script->hasBaselineScript())
        return Method_Skipped;

    if (script->isDebuggee() || (osrFrame && osrFrame->isDebuggee())) {
        TrackAndSpewIonAbort(cx, script, "debugging");
        return Method_Skipped;
    }

    if (!CheckScript(cx, script, bool(osrPc))) {
        JitSpew(JitSpew_IonAbort, "Aborted compilation of %s:%" PRIuSIZE, script->filename(), script->lineno());
        return Method_CantCompile;
    }

    MethodStatus status = CheckScriptSize(cx, script);
    if (status != Method_Compiled) {
        JitSpew(JitSpew_IonAbort, "Aborted compilation of %s:%" PRIuSIZE, script->filename(), script->lineno());
        return status;
    }

    bool recompile = false;
    OptimizationLevel optimizationLevel = GetOptimizationLevel(script, osrPc);
    if (optimizationLevel == OptimizationLevel::DontCompile)
        return Method_Skipped;

    if (!CanLikelyAllocateMoreExecutableMemory()) {
        script->resetWarmUpCounter();
        return Method_Skipped;
    }

    if (script->hasIonScript()) {
        IonScript* scriptIon = script->ionScript();
        if (!scriptIon->method())
            return Method_CantCompile;

        // Don't recompile/overwrite higher optimized code,
        // with a lower optimization level.
        if (optimizationLevel <= scriptIon->optimizationLevel() && !forceRecompile)
            return Method_Compiled;

        // Don't start compiling if already compiling
        if (scriptIon->isRecompiling())
            return Method_Compiled;

        if (osrPc)
            scriptIon->resetOsrPcMismatchCounter();

        recompile = true;
    }

    if (script->baselineScript()->hasPendingIonBuilder()) {
        IonBuilder* buildIon = script->baselineScript()->pendingIonBuilder();
        if (optimizationLevel <= buildIon->optimizationInfo().level() && !forceRecompile)
            return Method_Compiled;

        recompile = true;
    }

    AbortReason reason = IonCompile(cx, script, osrFrame, osrPc, recompile, optimizationLevel);
    if (reason == AbortReason_Error)
        return Method_Error;

    if (reason == AbortReason_Disable)
        return Method_CantCompile;

    if (reason == AbortReason_Alloc) {
        ReportOutOfMemory(cx);
        return Method_Error;
    }

    // Compilation succeeded or we invalidated right away or an inlining/alloc abort
    if (script->hasIonScript())
        return Method_Compiled;
    return Method_Skipped;
}

} // namespace jit
} // namespace js

bool
jit::OffThreadCompilationAvailable(JSContext* cx)
{
    // Even if off thread compilation is enabled, compilation must still occur
    // on the main thread in some cases.
    //
    // Require cpuCount > 1 so that Ion compilation jobs and main-thread
    // execution are not competing for the same resources.
    return cx->runtime()->canUseOffthreadIonCompilation()
        && HelperThreadState().cpuCount > 1
        && CanUseExtraThreads();
}

MethodStatus
jit::CanEnter(JSContext* cx, RunState& state)
{
    MOZ_ASSERT(jit::IsIonEnabled(cx));

    JSScript* script = state.script();

    // Skip if the script has been disabled.
    if (!script->canIonCompile())
        return Method_Skipped;

    // Skip if the script is being compiled off thread.
    if (script->isIonCompilingOffThread())
        return Method_Skipped;

    // Skip if the code is expected to result in a bailout.
    if (script->hasIonScript() && script->ionScript()->bailoutExpected())
        return Method_Skipped;

    RootedScript rscript(cx, script);

    // If constructing, allocate a new |this| object before building Ion.
    // Creating |this| is done before building Ion because it may change the
    // type information and invalidate compilation results.
    if (state.isInvoke()) {
        InvokeState& invoke = *state.asInvoke();

        if (TooManyActualArguments(invoke.args().length())) {
            TrackAndSpewIonAbort(cx, script, "too many actual args");
            ForbidCompilation(cx, script);
            return Method_CantCompile;
        }

        if (TooManyFormalArguments(invoke.args().callee().as<JSFunction>().nargs())) {
            TrackAndSpewIonAbort(cx, script, "too many args");
            ForbidCompilation(cx, script);
            return Method_CantCompile;
        }

        if (!state.maybeCreateThisForConstructor(cx)) {
            if (cx->isThrowingOutOfMemory()) {
                cx->recoverFromOutOfMemory();
                return Method_Skipped;
            }
            return Method_Error;
        }
    }

    // If --ion-eager is used, compile with Baseline first, so that we
    // can directly enter IonMonkey.
    if (JitOptions.eagerCompilation && !rscript->hasBaselineScript()) {
        MethodStatus status = CanEnterBaselineMethod(cx, state);
        if (status != Method_Compiled)
            return status;
    }

    // Skip if the script is being compiled off thread or can't be
    // Ion-compiled (again). MaybeCreateThisForConstructor could have
    // started an Ion compilation or marked the script as uncompilable.
    if (rscript->isIonCompilingOffThread() || !rscript->canIonCompile())
        return Method_Skipped;

    // Attempt compilation. Returns Method_Compiled if already compiled.
    MethodStatus status = Compile(cx, rscript, nullptr, nullptr);
    if (status != Method_Compiled) {
        if (status == Method_CantCompile)
            ForbidCompilation(cx, rscript);
        return status;
    }

    if (state.script()->baselineScript()->hasPendingIonBuilder()) {
        LinkIonScript(cx, state.script());
        if (!state.script()->hasIonScript())
            return jit::Method_Skipped;
    }

    return Method_Compiled;
}

static MethodStatus
BaselineCanEnterAtEntry(JSContext* cx, HandleScript script, BaselineFrame* frame)
{
    MOZ_ASSERT(jit::IsIonEnabled(cx));
    MOZ_ASSERT(frame->callee()->nonLazyScript()->canIonCompile());
    MOZ_ASSERT(!frame->callee()->nonLazyScript()->isIonCompilingOffThread());
    MOZ_ASSERT(!frame->callee()->nonLazyScript()->hasIonScript());
    MOZ_ASSERT(frame->isFunctionFrame());

    // Mark as forbidden if frame can't be handled.
    if (!CheckFrame(cx, frame)) {
        ForbidCompilation(cx, script);
        return Method_CantCompile;
    }

    // Attempt compilation. Returns Method_Compiled if already compiled.
    MethodStatus status = Compile(cx, script, frame, nullptr);
    if (status != Method_Compiled) {
        if (status == Method_CantCompile)
            ForbidCompilation(cx, script);
        return status;
    }

    return Method_Compiled;
}

// Decide if a transition from baseline execution to Ion code should occur.
// May compile or recompile the target JSScript.
static MethodStatus
BaselineCanEnterAtBranch(JSContext* cx, HandleScript script, BaselineFrame* osrFrame, jsbytecode* pc)
{
    MOZ_ASSERT(jit::IsIonEnabled(cx));
    MOZ_ASSERT((JSOp)*pc == JSOP_LOOPENTRY);
    MOZ_ASSERT(LoopEntryCanIonOsr(pc));

    // Skip if the script has been disabled.
    if (!script->canIonCompile())
        return Method_Skipped;

    // Skip if the script is being compiled off thread.
    if (script->isIonCompilingOffThread())
        return Method_Skipped;

    // Skip if the code is expected to result in a bailout.
    if (script->hasIonScript() && script->ionScript()->bailoutExpected())
        return Method_Skipped;

    // Optionally ignore on user request.
    if (!JitOptions.osr)
        return Method_Skipped;

    // Mark as forbidden if frame can't be handled.
    if (!CheckFrame(cx, osrFrame)) {
        ForbidCompilation(cx, script);
        return Method_CantCompile;
    }

    // Check if the jitcode still needs to get linked and do this
    // to have a valid IonScript.
    if (script->baselineScript()->hasPendingIonBuilder())
        LinkIonScript(cx, script);

    // By default a recompilation doesn't happen on osr mismatch.
    // Decide if we want to force a recompilation if this happens too much.
    bool force = false;
    if (script->hasIonScript() && pc != script->ionScript()->osrPc()) {
        uint32_t count = script->ionScript()->incrOsrPcMismatchCounter();
        if (count <= JitOptions.osrPcMismatchesBeforeRecompile)
            return Method_Skipped;
        force = true;
    }

    // Attempt compilation.
    // - Returns Method_Compiled if the right ionscript is present
    //   (Meaning it was present or a sequantial compile finished)
    // - Returns Method_Skipped if pc doesn't match
    //   (This means a background thread compilation with that pc could have started or not.)
    RootedScript rscript(cx, script);
    MethodStatus status = Compile(cx, rscript, osrFrame, pc, force);
    if (status != Method_Compiled) {
        if (status == Method_CantCompile)
            ForbidCompilation(cx, script);
        return status;
    }

    // Return the compilation was skipped when the osr pc wasn't adjusted.
    // This can happen when there was still an IonScript available and a
    // background compilation started, but hasn't finished yet.
    // Or when we didn't force a recompile.
    if (script->hasIonScript() && pc != script->ionScript()->osrPc())
        return Method_Skipped;

    return Method_Compiled;
}

bool
jit::IonCompileScriptForBaseline(JSContext* cx, BaselineFrame* frame, jsbytecode* pc)
{
    // A TI OOM will disable TI and Ion.
    if (!jit::IsIonEnabled(cx))
        return true;

    RootedScript script(cx, frame->script());
    bool isLoopEntry = JSOp(*pc) == JSOP_LOOPENTRY;

    MOZ_ASSERT(!isLoopEntry || LoopEntryCanIonOsr(pc));

    if (!script->canIonCompile()) {
        // TODO: ASSERT that ion-compilation-disabled checker stub doesn't exist.
        // TODO: Clear all optimized stubs.
        // TODO: Add a ion-compilation-disabled checker IC stub
        script->resetWarmUpCounter();
        return true;
    }

    MOZ_ASSERT(!script->isIonCompilingOffThread());

    // If Ion script exists, but PC is not at a loop entry, then Ion will be entered for
    // this script at an appropriate LOOPENTRY or the next time this function is called.
    if (script->hasIonScript() && !isLoopEntry) {
        JitSpew(JitSpew_BaselineOSR, "IonScript exists, but not at loop entry!");
        // TODO: ASSERT that a ion-script-already-exists checker stub doesn't exist.
        // TODO: Clear all optimized stubs.
        // TODO: Add a ion-script-already-exists checker stub.
        return true;
    }

    // Ensure that Ion-compiled code is available.
    JitSpew(JitSpew_BaselineOSR,
            "WarmUpCounter for %s:%" PRIuSIZE " reached %d at pc %p, trying to switch to Ion!",
            script->filename(), script->lineno(), (int) script->getWarmUpCount(), (void*) pc);

    MethodStatus stat;
    if (isLoopEntry) {
        MOZ_ASSERT(LoopEntryCanIonOsr(pc));
        JitSpew(JitSpew_BaselineOSR, "  Compile at loop entry!");
        stat = BaselineCanEnterAtBranch(cx, script, frame, pc);
    } else if (frame->isFunctionFrame()) {
        JitSpew(JitSpew_BaselineOSR, "  Compile function from top for later entry!");
        stat = BaselineCanEnterAtEntry(cx, script, frame);
    } else {
        return true;
    }

    if (stat == Method_Error) {
        JitSpew(JitSpew_BaselineOSR, "  Compile with Ion errored!");
        return false;
    }

    if (stat == Method_CantCompile)
        JitSpew(JitSpew_BaselineOSR, "  Can't compile with Ion!");
    else if (stat == Method_Skipped)
        JitSpew(JitSpew_BaselineOSR, "  Skipped compile with Ion!");
    else if (stat == Method_Compiled)
        JitSpew(JitSpew_BaselineOSR, "  Compiled with Ion!");
    else
        MOZ_CRASH("Invalid MethodStatus!");

    // Failed to compile.  Reset warm-up counter and return.
    if (stat != Method_Compiled) {
        // TODO: If stat == Method_CantCompile, insert stub that just skips the
        // warm-up counter entirely, instead of resetting it.
        bool bailoutExpected = script->hasIonScript() && script->ionScript()->bailoutExpected();
        if (stat == Method_CantCompile || bailoutExpected) {
            JitSpew(JitSpew_BaselineOSR, "  Reset WarmUpCounter cantCompile=%s bailoutExpected=%s!",
                    stat == Method_CantCompile ? "yes" : "no",
                    bailoutExpected ? "yes" : "no");
            script->resetWarmUpCounter();
        }
        return true;
    }

    return true;
}


MethodStatus
jit::Recompile(JSContext* cx, HandleScript script, BaselineFrame* osrFrame, jsbytecode* osrPc,
               bool force)
{
    MOZ_ASSERT(script->hasIonScript());
    if (script->ionScript()->isRecompiling())
        return Method_Compiled;

    MethodStatus status = Compile(cx, script, osrFrame, osrPc, force);
    if (status != Method_Compiled) {
        if (status == Method_CantCompile)
            ForbidCompilation(cx, script);
        return status;
    }

    return Method_Compiled;
}

MethodStatus
jit::CanEnterUsingFastInvoke(JSContext* cx, HandleScript script, uint32_t numActualArgs)
{
    MOZ_ASSERT(jit::IsIonEnabled(cx));

    // Skip if the code is expected to result in a bailout.
    if (!script->hasIonScript() || script->ionScript()->bailoutExpected())
        return Method_Skipped;

    // Don't handle arguments underflow, to make this work we would have to pad
    // missing arguments with |undefined|.
    if (numActualArgs < script->functionNonDelazifying()->nargs())
        return Method_Skipped;

    if (!cx->compartment()->ensureJitCompartmentExists(cx))
        return Method_Error;

    // This can GC, so afterward, script->ion is not guaranteed to be valid.
    if (!cx->runtime()->jitRuntime()->enterIon())
        return Method_Error;

    if (!script->hasIonScript())
        return Method_Skipped;

    return Method_Compiled;
}

static JitExecStatus
EnterIon(JSContext* cx, EnterJitData& data)
{
    JS_CHECK_RECURSION(cx, return JitExec_Aborted);
    MOZ_ASSERT(jit::IsIonEnabled(cx));
    MOZ_ASSERT(!data.osrFrame);

#ifdef DEBUG
    // See comment in EnterBaseline.
    mozilla::Maybe<JS::AutoAssertNoGC> nogc;
    nogc.emplace(cx);
#endif

    EnterJitCode enter = cx->runtime()->jitRuntime()->enterIon();

    // Caller must construct |this| before invoking the Ion function.
    MOZ_ASSERT_IF(data.constructing,
                  data.maxArgv[0].isObject() || data.maxArgv[0].isMagic(JS_UNINITIALIZED_LEXICAL));

    data.result.setInt32(data.numActualArgs);
    {
        AssertCompartmentUnchanged pcc(cx);
        ActivationEntryMonitor entryMonitor(cx, data.calleeToken);
        JitActivation activation(cx);

#ifdef DEBUG
        nogc.reset();
#endif
        CALL_GENERATED_CODE(enter, data.jitcode, data.maxArgc, data.maxArgv, /* osrFrame = */nullptr, data.calleeToken,
                            /* envChain = */ nullptr, 0, data.result.address());
    }

    MOZ_ASSERT(!cx->runtime()->jitRuntime()->hasIonReturnOverride());

    // Jit callers wrap primitive constructor return, except for derived class constructors.
    if (!data.result.isMagic() && data.constructing &&
        data.result.isPrimitive())
    {
        MOZ_ASSERT(data.maxArgv[0].isObject());
        data.result = data.maxArgv[0];
    }

    // Release temporary buffer used for OSR into Ion.
    cx->runtime()->getJitRuntime(cx)->freeOsrTempData();

    MOZ_ASSERT_IF(data.result.isMagic(), data.result.isMagic(JS_ION_ERROR));
    return data.result.isMagic() ? JitExec_Error : JitExec_Ok;
}

bool
jit::SetEnterJitData(JSContext* cx, EnterJitData& data, RunState& state,
                     MutableHandle<GCVector<Value>> vals)
{
    data.osrFrame = nullptr;

    if (state.isInvoke()) {
        const CallArgs& args = state.asInvoke()->args();
        unsigned numFormals = state.script()->functionNonDelazifying()->nargs();
        data.constructing = state.asInvoke()->constructing();
        data.numActualArgs = args.length();
        data.maxArgc = Max(args.length(), numFormals) + 1;
        data.envChain = nullptr;
        data.calleeToken = CalleeToToken(&args.callee().as<JSFunction>(), data.constructing);

        if (data.numActualArgs >= numFormals) {
            data.maxArgv = args.base() + 1;
        } else {
            MOZ_ASSERT(vals.empty());
            unsigned numPushedArgs = Max(args.length(), numFormals);
            if (!vals.reserve(numPushedArgs + 1 + data.constructing))
                return false;

            // Append |this| and any provided arguments.
            for (size_t i = 1; i < args.length() + 2; ++i)
                vals.infallibleAppend(args.base()[i]);

            // Pad missing arguments with |undefined|.
            while (vals.length() < numFormals + 1)
                vals.infallibleAppend(UndefinedValue());

            if (data.constructing)
                vals.infallibleAppend(args.newTarget());

            MOZ_ASSERT(vals.length() >= numFormals + 1 + data.constructing);
            data.maxArgv = vals.begin();
        }
    } else {
        data.constructing = false;
        data.numActualArgs = 0;
        data.maxArgc = 0;
        data.maxArgv = nullptr;
        data.envChain = state.asExecute()->environmentChain();

        data.calleeToken = CalleeToToken(state.script());

        if (state.script()->isForEval() && state.script()->isDirectEvalInFunction()) {
            // Push newTarget onto the stack.
            if (!vals.reserve(1))
                return false;

            data.maxArgc = 1;
            data.maxArgv = vals.begin();
            if (state.asExecute()->newTarget().isNull()) {
                ScriptFrameIter iter(cx);
                vals.infallibleAppend(iter.newTarget());
            } else {
                vals.infallibleAppend(state.asExecute()->newTarget());
            }
        }
    }

    return true;
}

JitExecStatus
jit::IonCannon(JSContext* cx, RunState& state)
{
    IonScript* ion = state.script()->ionScript();

    EnterJitData data(cx);
    data.jitcode = ion->method()->raw();

    Rooted<GCVector<Value>> vals(cx, GCVector<Value>(cx));
    if (!SetEnterJitData(cx, data, state, &vals))
        return JitExec_Error;

    JitExecStatus status = EnterIon(cx, data);

    if (status == JitExec_Ok)
        state.setReturnValue(data.result);

    return status;
}

JitExecStatus
jit::FastInvoke(JSContext* cx, HandleFunction fun, CallArgs& args)
{
    JS_CHECK_RECURSION(cx, return JitExec_Error);

    RootedScript script(cx, fun->nonLazyScript());

    if (!Debugger::checkNoExecute(cx, script))
        return JitExec_Error;

#ifdef DEBUG
    // See comment in EnterBaseline.
    mozilla::Maybe<JS::AutoAssertNoGC> nogc;
    nogc.emplace(cx);
#endif

    IonScript* ion = script->ionScript();
    JitCode* code = ion->method();
    void* jitcode = code->raw();

    MOZ_ASSERT(jit::IsIonEnabled(cx));
    MOZ_ASSERT(!ion->bailoutExpected());

    ActivationEntryMonitor entryMonitor(cx, CalleeToToken(script));
    JitActivation activation(cx);

    EnterJitCode enter = cx->runtime()->jitRuntime()->enterIon();
    void* calleeToken = CalleeToToken(fun, /* constructing = */ false);

    RootedValue result(cx, Int32Value(args.length()));
    MOZ_ASSERT(args.length() >= fun->nargs());

#ifdef DEBUG
    nogc.reset();
#endif
    CALL_GENERATED_CODE(enter, jitcode, args.length() + 1, args.array() - 1, /* osrFrame = */nullptr,
                        calleeToken, /* envChain = */ nullptr, 0, result.address());

    MOZ_ASSERT(!cx->runtime()->jitRuntime()->hasIonReturnOverride());

    args.rval().set(result);

    MOZ_ASSERT_IF(result.isMagic(), result.isMagic(JS_ION_ERROR));
    return result.isMagic() ? JitExec_Error : JitExec_Ok;
}

static void
InvalidateActivation(FreeOp* fop, const JitActivationIterator& activations, bool invalidateAll)
{
    JitSpew(JitSpew_IonInvalidate, "BEGIN invalidating activation");

#ifdef CHECK_OSIPOINT_REGISTERS
    if (JitOptions.checkOsiPointRegisters)
        activations->asJit()->setCheckRegs(false);
#endif

    size_t frameno = 1;

    for (JitFrameIterator it(activations); !it.done(); ++it, ++frameno) {
        MOZ_ASSERT_IF(frameno == 1, it.isExitFrame() || it.type() == JitFrame_Bailout);

#ifdef JS_JITSPEW
        switch (it.type()) {
          case JitFrame_Exit:
            JitSpew(JitSpew_IonInvalidate, "#%" PRIuSIZE " exit frame @ %p", frameno, it.fp());
            break;
          case JitFrame_BaselineJS:
          case JitFrame_IonJS:
          case JitFrame_Bailout:
          {
            MOZ_ASSERT(it.isScripted());
            const char* type = "Unknown";
            if (it.isIonJS())
                type = "Optimized";
            else if (it.isBaselineJS())
                type = "Baseline";
            else if (it.isBailoutJS())
                type = "Bailing";
            JitSpew(JitSpew_IonInvalidate,
                    "#%" PRIuSIZE " %s JS frame @ %p, %s:%" PRIuSIZE " (fun: %p, script: %p, pc %p)",
                    frameno, type, it.fp(), it.script()->maybeForwardedFilename(),
                    it.script()->lineno(), it.maybeCallee(), (JSScript*)it.script(),
                    it.returnAddressToFp());
            break;
          }
          case JitFrame_IonStub:
            JitSpew(JitSpew_IonInvalidate, "#%" PRIuSIZE " ion stub frame @ %p", frameno, it.fp());
            break;
          case JitFrame_BaselineStub:
            JitSpew(JitSpew_IonInvalidate, "#%" PRIuSIZE " baseline stub frame @ %p", frameno, it.fp());
            break;
          case JitFrame_Rectifier:
            JitSpew(JitSpew_IonInvalidate, "#%" PRIuSIZE " rectifier frame @ %p", frameno, it.fp());
            break;
          case JitFrame_IonAccessorIC:
            JitSpew(JitSpew_IonInvalidate, "#%" PRIuSIZE " ion IC getter/setter frame @ %p", frameno, it.fp());
            break;
          case JitFrame_Entry:
            JitSpew(JitSpew_IonInvalidate, "#%" PRIuSIZE " entry frame @ %p", frameno, it.fp());
            break;
        }
#endif // JS_JITSPEW

        if (!it.isIonScripted())
            continue;

        bool calledFromLinkStub = false;
        JitCode* lazyLinkStub = fop->runtime()->jitRuntime()->lazyLinkStub();
        if (it.returnAddressToFp() >= lazyLinkStub->raw() &&
            it.returnAddressToFp() < lazyLinkStub->rawEnd())
        {
            calledFromLinkStub = true;
        }

        // See if the frame has already been invalidated.
        if (!calledFromLinkStub && it.checkInvalidation())
            continue;

        JSScript* script = it.script();
        if (!script->hasIonScript())
            continue;

        if (!invalidateAll && !script->ionScript()->invalidated())
            continue;

        IonScript* ionScript = script->ionScript();

        // Purge ICs before we mark this script as invalidated. This will
        // prevent lastJump_ from appearing to be a bogus pointer, just
        // in case anyone tries to read it.
        ionScript->purgeCaches();
        ionScript->purgeOptimizedStubs(script->zone());

        // Clean up any pointers from elsewhere in the runtime to this IonScript
        // which is about to become disconnected from its JSScript.
        ionScript->unlinkFromRuntime(fop);

        // This frame needs to be invalidated. We do the following:
        //
        // 1. Increment the reference counter to keep the ionScript alive
        //    for the invalidation bailout or for the exception handler.
        // 2. Determine safepoint that corresponds to the current call.
        // 3. From safepoint, get distance to the OSI-patchable offset.
        // 4. From the IonScript, determine the distance between the
        //    call-patchable offset and the invalidation epilogue.
        // 5. Patch the OSI point with a call-relative to the
        //    invalidation epilogue.
        //
        // The code generator ensures that there's enough space for us
        // to patch in a call-relative operation at each invalidation
        // point.
        //
        // Note: you can't simplify this mechanism to "just patch the
        // instruction immediately after the call" because things may
        // need to move into a well-defined register state (using move
        // instructions after the call) in to capture an appropriate
        // snapshot after the call occurs.

        ionScript->incrementInvalidationCount();

        JitCode* ionCode = ionScript->method();

        JS::Zone* zone = script->zone();
        if (zone->needsIncrementalBarrier()) {
            // We're about to remove edges from the JSScript to gcthings
            // embedded in the JitCode. Perform one final trace of the
            // JitCode for the incremental GC, as it must know about
            // those edges.
            ionCode->traceChildren(zone->barrierTracer());
        }
        ionCode->setInvalidated();

        // Don't adjust OSI points in the linkStub (which don't exist), or in a
        // bailout path.
        if (calledFromLinkStub || it.isBailoutJS())
            continue;

        // Write the delta (from the return address offset to the
        // IonScript pointer embedded into the invalidation epilogue)
        // where the safepointed call instruction used to be. We rely on
        // the call sequence causing the safepoint being >= the size of
        // a uint32, which is checked during safepoint index
        // construction.
        AutoWritableJitCode awjc(ionCode);
        const SafepointIndex* si = ionScript->getSafepointIndex(it.returnAddressToFp());
        CodeLocationLabel dataLabelToMunge(it.returnAddressToFp());
        ptrdiff_t delta = ionScript->invalidateEpilogueDataOffset() -
                          (it.returnAddressToFp() - ionCode->raw());
        Assembler::PatchWrite_Imm32(dataLabelToMunge, Imm32(delta));

        CodeLocationLabel osiPatchPoint = SafepointReader::InvalidationPatchPoint(ionScript, si);
        CodeLocationLabel invalidateEpilogue(ionCode, CodeOffset(ionScript->invalidateEpilogueOffset()));

        JitSpew(JitSpew_IonInvalidate, "   ! Invalidate ionScript %p (inv count %" PRIuSIZE ") -> patching osipoint %p",
                ionScript, ionScript->invalidationCount(), (void*) osiPatchPoint.raw());
        Assembler::PatchWrite_NearCall(osiPatchPoint, invalidateEpilogue);
    }

    JitSpew(JitSpew_IonInvalidate, "END invalidating activation");
}

void
jit::InvalidateAll(FreeOp* fop, Zone* zone)
{
    // The caller should previously have cancelled off thread compilation.
#ifdef DEBUG
    for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next())
        MOZ_ASSERT(!HasOffThreadIonCompile(comp));
#endif

    for (JitActivationIterator iter(fop->runtime()); !iter.done(); ++iter) {
        if (iter->compartment()->zone() == zone) {
            JitSpew(JitSpew_IonInvalidate, "Invalidating all frames for GC");
            InvalidateActivation(fop, iter, true);
        }
    }
}


void
jit::Invalidate(TypeZone& types, FreeOp* fop,
                const RecompileInfoVector& invalid, bool resetUses,
                bool cancelOffThread)
{
    JitSpew(JitSpew_IonInvalidate, "Start invalidation.");

    // Add an invalidation reference to all invalidated IonScripts to indicate
    // to the traversal which frames have been invalidated.
    size_t numInvalidations = 0;
    for (size_t i = 0; i < invalid.length(); i++) {
        const CompilerOutput* co = invalid[i].compilerOutput(types);
        if (!co)
            continue;
        MOZ_ASSERT(co->isValid());

        if (cancelOffThread)
            CancelOffThreadIonCompile(co->script());

        if (!co->ion())
            continue;

        JitSpew(JitSpew_IonInvalidate, " Invalidate %s:%" PRIuSIZE ", IonScript %p",
                co->script()->filename(), co->script()->lineno(), co->ion());

        // Keep the ion script alive during the invalidation and flag this
        // ionScript as being invalidated.  This increment is removed by the
        // loop after the calls to InvalidateActivation.
        co->ion()->incrementInvalidationCount();
        numInvalidations++;
    }

    if (!numInvalidations) {
        JitSpew(JitSpew_IonInvalidate, " No IonScript invalidation.");
        return;
    }

    for (JitActivationIterator iter(fop->runtime()); !iter.done(); ++iter)
        InvalidateActivation(fop, iter, false);

    // Drop the references added above. If a script was never active, its
    // IonScript will be immediately destroyed. Otherwise, it will be held live
    // until its last invalidated frame is destroyed.
    for (size_t i = 0; i < invalid.length(); i++) {
        CompilerOutput* co = invalid[i].compilerOutput(types);
        if (!co)
            continue;
        MOZ_ASSERT(co->isValid());

        JSScript* script = co->script();
        IonScript* ionScript = co->ion();
        if (!ionScript)
            continue;

        script->setIonScript(nullptr, nullptr);
        ionScript->decrementInvalidationCount(fop);
        co->invalidate();
        numInvalidations--;

        // Wait for the scripts to get warm again before doing another
        // compile, unless we are recompiling *because* a script got hot
        // (resetUses is false).
        if (resetUses)
            script->resetWarmUpCounter();
    }

    // Make sure we didn't leak references by invalidating the same IonScript
    // multiple times in the above loop.
    MOZ_ASSERT(!numInvalidations);
}

void
jit::Invalidate(JSContext* cx, const RecompileInfoVector& invalid, bool resetUses,
                bool cancelOffThread)
{
    jit::Invalidate(cx->zone()->types, cx->runtime()->defaultFreeOp(), invalid, resetUses,
                    cancelOffThread);
}

void
jit::IonScript::invalidate(JSContext* cx, bool resetUses, const char* reason)
{
    JitSpew(JitSpew_IonInvalidate, " Invalidate IonScript %p: %s", this, reason);

    // RecompileInfoVector has inline space for at least one element.
    RecompileInfoVector list;
    MOZ_RELEASE_ASSERT(list.reserve(1));
    list.infallibleAppend(recompileInfo());

    Invalidate(cx, list, resetUses, true);
}

void
jit::Invalidate(JSContext* cx, JSScript* script, bool resetUses, bool cancelOffThread)
{
    MOZ_ASSERT(script->hasIonScript());

    if (cx->runtime()->spsProfiler.enabled()) {
        // Register invalidation with profiler.
        // Format of event payload string:
        //      "<filename>:<lineno>"

        // Get the script filename, if any, and its length.
        const char* filename = script->filename();
        if (filename == nullptr)
            filename = "<unknown>";

        // Construct the descriptive string.
        char* buf = JS_smprintf("Invalidate %s:%" PRIuSIZE, filename, script->lineno());

        // Ignore the event on allocation failure.
        if (buf) {
            cx->runtime()->spsProfiler.markEvent(buf);
            JS_smprintf_free(buf);
        }
    }

    // RecompileInfoVector has inline space for at least one element.
    RecompileInfoVector scripts;
    MOZ_ASSERT(script->hasIonScript());
    MOZ_RELEASE_ASSERT(scripts.reserve(1));
    scripts.infallibleAppend(script->ionScript()->recompileInfo());

    Invalidate(cx, scripts, resetUses, cancelOffThread);
}

static void
FinishInvalidationOf(FreeOp* fop, JSScript* script, IonScript* ionScript)
{
    TypeZone& types = script->zone()->types;

    // Note: If the script is about to be swept, the compiler output may have
    // already been destroyed.
    if (CompilerOutput* output = ionScript->recompileInfo().compilerOutput(types))
        output->invalidate();

    // If this script has Ion code on the stack, invalidated() will return
    // true. In this case we have to wait until destroying it.
    if (!ionScript->invalidated())
        jit::IonScript::Destroy(fop, ionScript);
}

void
jit::FinishInvalidation(FreeOp* fop, JSScript* script)
{
    // In all cases, nullptr out script->ion to avoid re-entry.
    if (script->hasIonScript()) {
        IonScript* ion = script->ionScript();
        script->setIonScript(nullptr, nullptr);
        FinishInvalidationOf(fop, script, ion);
    }
}

void
jit::ForbidCompilation(JSContext* cx, JSScript* script)
{
    JitSpew(JitSpew_IonAbort, "Disabling Ion compilation of script %s:%" PRIuSIZE,
            script->filename(), script->lineno());

    CancelOffThreadIonCompile(script);

    if (script->hasIonScript())
        Invalidate(cx, script, false);

    script->setIonScript(cx->runtime(), ION_DISABLED_SCRIPT);
}

AutoFlushICache*
PerThreadData::autoFlushICache() const
{
    return autoFlushICache_;
}

void
PerThreadData::setAutoFlushICache(AutoFlushICache* afc)
{
    autoFlushICache_ = afc;
}

// Set the range for the merging of flushes.  The flushing is deferred until the end of
// the AutoFlushICache context.  Subsequent flushing within this range will is also
// deferred.  This is only expected to be defined once for each AutoFlushICache
// context.  It assumes the range will be flushed is required to be within an
// AutoFlushICache context.
void
AutoFlushICache::setRange(uintptr_t start, size_t len)
{
#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
    AutoFlushICache* afc = TlsPerThreadData.get()->PerThreadData::autoFlushICache();
    MOZ_ASSERT(afc);
    MOZ_ASSERT(!afc->start_);
    JitSpewCont(JitSpew_CacheFlush, "(%" PRIxPTR " %" PRIxSIZE "):", start, len);

    uintptr_t stop = start + len;
    afc->start_ = start;
    afc->stop_ = stop;
#endif
}

// Flush the instruction cache.
//
// If called within a dynamic AutoFlushICache context and if the range is already pending
// flushing for this AutoFlushICache context then the request is ignored with the
// understanding that it will be flushed on exit from the AutoFlushICache context.
// Otherwise the range is flushed immediately.
//
// Updates outside the current code object are typically the exception so they are flushed
// immediately rather than attempting to merge them.
//
// For efficiency it is expected that all large ranges will be flushed within an
// AutoFlushICache, so check.  If this assertion is hit then it does not necessarily
// indicate a program fault but it might indicate a lost opportunity to merge cache
// flushing.  It can be corrected by wrapping the call in an AutoFlushICache to context.
//
// Note this can be called without TLS PerThreadData defined so this case needs
// to be guarded against. E.g. when patching instructions from the exception
// handler on MacOS running the ARM simulator.
void
AutoFlushICache::flush(uintptr_t start, size_t len)
{
#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
    PerThreadData* pt = TlsPerThreadData.get();
    AutoFlushICache* afc = pt ? pt->PerThreadData::autoFlushICache() : nullptr;
    if (!afc) {
        JitSpewCont(JitSpew_CacheFlush, "#");
        ExecutableAllocator::cacheFlush((void*)start, len);
        MOZ_ASSERT(len <= 32);
        return;
    }

    uintptr_t stop = start + len;
    if (start >= afc->start_ && stop <= afc->stop_) {
        // Update is within the pending flush range, so defer to the end of the context.
        JitSpewCont(JitSpew_CacheFlush, afc->inhibit_ ? "-" : "=");
        return;
    }

    JitSpewCont(JitSpew_CacheFlush, afc->inhibit_ ? "x" : "*");
    ExecutableAllocator::cacheFlush((void*)start, len);
#endif
}

// Flag the current dynamic AutoFlushICache as inhibiting flushing. Useful in error paths
// where the changes are being abandoned.
void
AutoFlushICache::setInhibit()
{
#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
    AutoFlushICache* afc = TlsPerThreadData.get()->PerThreadData::autoFlushICache();
    MOZ_ASSERT(afc);
    MOZ_ASSERT(afc->start_);
    JitSpewCont(JitSpew_CacheFlush, "I");
    afc->inhibit_ = true;
#endif
}

// The common use case is merging cache flushes when preparing a code object.  In this
// case the entire range of the code object is being flushed and as the code is patched
// smaller redundant flushes could occur.  The design allows an AutoFlushICache dynamic
// thread local context to be declared in which the range of the code object can be set
// which defers flushing until the end of this dynamic context.  The redundant flushing
// within this code range is also deferred avoiding redundant flushing.  Flushing outside
// this code range is not affected and proceeds immediately.
//
// In some cases flushing is not necessary, such as when compiling an wasm module which
// is flushed again when dynamically linked, and also in error paths that abandon the
// code.  Flushing within the set code range can be inhibited within the AutoFlushICache
// dynamic context by setting an inhibit flag.
//
// The JS compiler can be re-entered while within an AutoFlushICache dynamic context and
// it is assumed that code being assembled or patched is not executed before the exit of
// the respective AutoFlushICache dynamic context.
//
AutoFlushICache::AutoFlushICache(const char* nonce, bool inhibit)
#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
  : start_(0),
    stop_(0),
    name_(nonce),
    inhibit_(inhibit)
#endif
{
#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
    PerThreadData* pt = TlsPerThreadData.get();
    AutoFlushICache* afc = pt->PerThreadData::autoFlushICache();
    if (afc)
        JitSpew(JitSpew_CacheFlush, "<%s,%s%s ", nonce, afc->name_, inhibit ? " I" : "");
    else
        JitSpewCont(JitSpew_CacheFlush, "<%s%s ", nonce, inhibit ? " I" : "");

    prev_ = afc;
    pt->PerThreadData::setAutoFlushICache(this);
#endif
}

AutoFlushICache::~AutoFlushICache()
{
#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
    PerThreadData* pt = TlsPerThreadData.get();
    MOZ_ASSERT(pt->PerThreadData::autoFlushICache() == this);

    if (!inhibit_ && start_)
        ExecutableAllocator::cacheFlush((void*)start_, size_t(stop_ - start_));

    JitSpewCont(JitSpew_CacheFlush, "%s%s>", name_, start_ ? "" : " U");
    JitSpewFin(JitSpew_CacheFlush);
    pt->PerThreadData::setAutoFlushICache(prev_);
#endif
}

void
jit::PurgeCaches(JSScript* script)
{
    if (script->hasIonScript())
        script->ionScript()->purgeCaches();
}

size_t
jit::SizeOfIonData(JSScript* script, mozilla::MallocSizeOf mallocSizeOf)
{
    size_t result = 0;

    if (script->hasIonScript())
        result += script->ionScript()->sizeOfIncludingThis(mallocSizeOf);

    return result;
}

void
jit::DestroyJitScripts(FreeOp* fop, JSScript* script)
{
    if (script->hasIonScript())
        jit::IonScript::Destroy(fop, script->ionScript());

    if (script->hasBaselineScript())
        jit::BaselineScript::Destroy(fop, script->baselineScript());
}

void
jit::TraceJitScripts(JSTracer* trc, JSScript* script)
{
    if (script->hasIonScript())
        jit::IonScript::Trace(trc, script->ionScript());

    if (script->hasBaselineScript())
        jit::BaselineScript::Trace(trc, script->baselineScript());
}

bool
jit::JitSupportsFloatingPoint()
{
    return js::jit::MacroAssembler::SupportsFloatingPoint();
}

bool
jit::JitSupportsUnalignedAccesses()
{
    return js::jit::MacroAssembler::SupportsUnalignedAccesses();
}

bool
jit::JitSupportsSimd()
{
    return js::jit::MacroAssembler::SupportsSimd();
}

bool
jit::JitSupportsAtomics()
{
#if defined(JS_CODEGEN_ARM)
    // Bug 1146902, bug 1077318: Enable Ion inlining of Atomics
    // operations on ARM only when the CPU has byte, halfword, and
    // doubleword load-exclusive and store-exclusive instructions,
    // until we can add support for systems that don't have those.
    return js::jit::HasLDSTREXBHD();
#else
    return true;
#endif
}

// If you change these, please also change the comment in TempAllocator.
/* static */ const size_t TempAllocator::BallastSize            = 16 * 1024;
/* static */ const size_t TempAllocator::PreferredLifoChunkSize = 32 * 1024;