diff options
Diffstat (limited to 'js/src/gc')
35 files changed, 17641 insertions, 0 deletions
diff --git a/js/src/gc/Allocator.cpp b/js/src/gc/Allocator.cpp new file mode 100644 index 000000000..f7dc50d02 --- /dev/null +++ b/js/src/gc/Allocator.cpp @@ -0,0 +1,628 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/Allocator.h" + +#include "jscntxt.h" + +#include "gc/GCInternals.h" +#include "gc/GCTrace.h" +#include "gc/Nursery.h" +#include "jit/JitCompartment.h" +#include "vm/Runtime.h" +#include "vm/String.h" + +#include "jsobjinlines.h" + +#include "gc/Heap-inl.h" + +using namespace js; +using namespace gc; + +template <typename T, AllowGC allowGC /* = CanGC */> +JSObject* +js::Allocate(ExclusiveContext* cx, AllocKind kind, size_t nDynamicSlots, InitialHeap heap, + const Class* clasp) +{ + static_assert(mozilla::IsConvertible<T*, JSObject*>::value, "must be JSObject derived"); + MOZ_ASSERT(IsObjectAllocKind(kind)); + size_t thingSize = Arena::thingSize(kind); + + MOZ_ASSERT(thingSize == Arena::thingSize(kind)); + MOZ_ASSERT(thingSize >= sizeof(JSObject_Slots0)); + static_assert(sizeof(JSObject_Slots0) >= CellSize, + "All allocations must be at least the allocator-imposed minimum size."); + + MOZ_ASSERT_IF(nDynamicSlots != 0, clasp->isNative() || clasp->isProxy()); + + // Off-main-thread alloc cannot trigger GC or make runtime assertions. + if (!cx->isJSContext()) + return GCRuntime::tryNewTenuredObject<NoGC>(cx, kind, thingSize, nDynamicSlots); + + JSContext* ncx = cx->asJSContext(); + JSRuntime* rt = ncx->runtime(); + if (!rt->gc.checkAllocatorState<allowGC>(ncx, kind)) + return nullptr; + + if (ncx->nursery().isEnabled() && heap != TenuredHeap) { + JSObject* obj = rt->gc.tryNewNurseryObject<allowGC>(ncx, thingSize, nDynamicSlots, clasp); + if (obj) + return obj; + + // Our most common non-jit allocation path is NoGC; thus, if we fail the + // alloc and cannot GC, we *must* return nullptr here so that the caller + // will do a CanGC allocation to clear the nursery. Failing to do so will + // cause all allocations on this path to land in Tenured, and we will not + // get the benefit of the nursery. + if (!allowGC) + return nullptr; + } + + return GCRuntime::tryNewTenuredObject<allowGC>(cx, kind, thingSize, nDynamicSlots); +} +template JSObject* js::Allocate<JSObject, NoGC>(ExclusiveContext* cx, gc::AllocKind kind, + size_t nDynamicSlots, gc::InitialHeap heap, + const Class* clasp); +template JSObject* js::Allocate<JSObject, CanGC>(ExclusiveContext* cx, gc::AllocKind kind, + size_t nDynamicSlots, gc::InitialHeap heap, + const Class* clasp); + +// Attempt to allocate a new GC thing out of the nursery. If there is not enough +// room in the nursery or there is an OOM, this method will return nullptr. +template <AllowGC allowGC> +JSObject* +GCRuntime::tryNewNurseryObject(JSContext* cx, size_t thingSize, size_t nDynamicSlots, const Class* clasp) +{ + MOZ_ASSERT(isNurseryAllocAllowed()); + MOZ_ASSERT(!cx->zone()->usedByExclusiveThread); + MOZ_ASSERT(!IsAtomsCompartment(cx->compartment())); + JSObject* obj = nursery.allocateObject(cx, thingSize, nDynamicSlots, clasp); + if (obj) + return obj; + + if (allowGC && !rt->mainThread.suppressGC) { + minorGC(JS::gcreason::OUT_OF_NURSERY); + + // Exceeding gcMaxBytes while tenuring can disable the Nursery. + if (nursery.isEnabled()) { + JSObject* obj = nursery.allocateObject(cx, thingSize, nDynamicSlots, clasp); + MOZ_ASSERT(obj); + return obj; + } + } + return nullptr; +} + +template <AllowGC allowGC> +JSObject* +GCRuntime::tryNewTenuredObject(ExclusiveContext* cx, AllocKind kind, size_t thingSize, + size_t nDynamicSlots) +{ + HeapSlot* slots = nullptr; + if (nDynamicSlots) { + slots = cx->zone()->pod_malloc<HeapSlot>(nDynamicSlots); + if (MOZ_UNLIKELY(!slots)) { + if (allowGC) + ReportOutOfMemory(cx); + return nullptr; + } + Debug_SetSlotRangeToCrashOnTouch(slots, nDynamicSlots); + } + + JSObject* obj = tryNewTenuredThing<JSObject, allowGC>(cx, kind, thingSize); + + if (obj) + obj->setInitialSlotsMaybeNonNative(slots); + else + js_free(slots); + + return obj; +} + +template <typename T, AllowGC allowGC /* = CanGC */> +T* +js::Allocate(ExclusiveContext* cx) +{ + static_assert(!mozilla::IsConvertible<T*, JSObject*>::value, "must not be JSObject derived"); + static_assert(sizeof(T) >= CellSize, + "All allocations must be at least the allocator-imposed minimum size."); + + AllocKind kind = MapTypeToFinalizeKind<T>::kind; + size_t thingSize = sizeof(T); + MOZ_ASSERT(thingSize == Arena::thingSize(kind)); + + if (cx->isJSContext()) { + JSContext* ncx = cx->asJSContext(); + if (!ncx->runtime()->gc.checkAllocatorState<allowGC>(ncx, kind)) + return nullptr; + } + + return GCRuntime::tryNewTenuredThing<T, allowGC>(cx, kind, thingSize); +} + +#define DECL_ALLOCATOR_INSTANCES(allocKind, traceKind, type, sizedType) \ + template type* js::Allocate<type, NoGC>(ExclusiveContext* cx);\ + template type* js::Allocate<type, CanGC>(ExclusiveContext* cx); +FOR_EACH_NONOBJECT_ALLOCKIND(DECL_ALLOCATOR_INSTANCES) +#undef DECL_ALLOCATOR_INSTANCES + +template <typename T, AllowGC allowGC> +/* static */ T* +GCRuntime::tryNewTenuredThing(ExclusiveContext* cx, AllocKind kind, size_t thingSize) +{ + // Bump allocate in the arena's current free-list span. + T* t = reinterpret_cast<T*>(cx->arenas()->allocateFromFreeList(kind, thingSize)); + if (MOZ_UNLIKELY(!t)) { + // Get the next available free list and allocate out of it. This may + // acquire a new arena, which will lock the chunk list. If there are no + // chunks available it may also allocate new memory directly. + t = reinterpret_cast<T*>(refillFreeListFromAnyThread(cx, kind, thingSize)); + + if (MOZ_UNLIKELY(!t && allowGC && cx->isJSContext())) { + // We have no memory available for a new chunk; perform an + // all-compartments, non-incremental, shrinking GC and wait for + // sweeping to finish. + JS::PrepareForFullGC(cx->asJSContext()); + AutoKeepAtoms keepAtoms(cx->perThreadData); + cx->asJSContext()->gc.gc(GC_SHRINK, JS::gcreason::LAST_DITCH); + cx->asJSContext()->gc.waitBackgroundSweepOrAllocEnd(); + + t = tryNewTenuredThing<T, NoGC>(cx, kind, thingSize); + if (!t) + ReportOutOfMemory(cx); + } + } + + checkIncrementalZoneState(cx, t); + TraceTenuredAlloc(t, kind); + return t; +} + +template <AllowGC allowGC> +bool +GCRuntime::checkAllocatorState(JSContext* cx, AllocKind kind) +{ + if (allowGC) { + if (!gcIfNeededPerAllocation(cx)) + return false; + } + +#if defined(JS_GC_ZEAL) || defined(DEBUG) + MOZ_ASSERT_IF(cx->compartment()->isAtomsCompartment(), + kind == AllocKind::ATOM || + kind == AllocKind::FAT_INLINE_ATOM || + kind == AllocKind::SYMBOL || + kind == AllocKind::JITCODE || + kind == AllocKind::SCOPE); + MOZ_ASSERT_IF(!cx->compartment()->isAtomsCompartment(), + kind != AllocKind::ATOM && + kind != AllocKind::FAT_INLINE_ATOM); + MOZ_ASSERT(!rt->isHeapBusy()); + MOZ_ASSERT(isAllocAllowed()); +#endif + + // Crash if we perform a GC action when it is not safe. + if (allowGC && !rt->mainThread.suppressGC) + rt->gc.verifyIsSafeToGC(); + + // For testing out of memory conditions + if (js::oom::ShouldFailWithOOM()) { + // If we are doing a fallible allocation, percolate up the OOM + // instead of reporting it. + if (allowGC) + ReportOutOfMemory(cx); + return false; + } + + return true; +} + +bool +GCRuntime::gcIfNeededPerAllocation(JSContext* cx) +{ +#ifdef JS_GC_ZEAL + if (needZealousGC()) + runDebugGC(); +#endif + + // Invoking the interrupt callback can fail and we can't usefully + // handle that here. Just check in case we need to collect instead. + if (rt->hasPendingInterrupt()) + gcIfRequested(); + + // If we have grown past our GC heap threshold while in the middle of + // an incremental GC, we're growing faster than we're GCing, so stop + // the world and do a full, non-incremental GC right now, if possible. + if (isIncrementalGCInProgress() && + cx->zone()->usage.gcBytes() > cx->zone()->threshold.gcTriggerBytes()) + { + PrepareZoneForGC(cx->zone()); + AutoKeepAtoms keepAtoms(cx->perThreadData); + gc(GC_NORMAL, JS::gcreason::INCREMENTAL_TOO_SLOW); + } + + return true; +} + +template <typename T> +/* static */ void +GCRuntime::checkIncrementalZoneState(ExclusiveContext* cx, T* t) +{ +#ifdef DEBUG + if (!cx->isJSContext()) + return; + + Zone* zone = cx->asJSContext()->zone(); + MOZ_ASSERT_IF(t && zone->wasGCStarted() && (zone->isGCMarking() || zone->isGCSweeping()), + t->asTenured().arena()->allocatedDuringIncremental); +#endif +} + + +// /////////// Arena -> Thing Allocator ////////////////////////////////////// + +void +GCRuntime::startBackgroundAllocTaskIfIdle() +{ + AutoLockHelperThreadState helperLock; + if (allocTask.isRunningWithLockHeld(helperLock)) + return; + + // Join the previous invocation of the task. This will return immediately + // if the thread has never been started. + allocTask.joinWithLockHeld(helperLock); + allocTask.startWithLockHeld(helperLock); +} + +/* static */ TenuredCell* +GCRuntime::refillFreeListFromAnyThread(ExclusiveContext* cx, AllocKind thingKind, size_t thingSize) +{ + cx->arenas()->checkEmptyFreeList(thingKind); + + if (cx->isJSContext()) + return refillFreeListFromMainThread(cx->asJSContext(), thingKind, thingSize); + + return refillFreeListOffMainThread(cx, thingKind); +} + +/* static */ TenuredCell* +GCRuntime::refillFreeListFromMainThread(JSContext* cx, AllocKind thingKind, size_t thingSize) +{ + // It should not be possible to allocate on the main thread while we are + // inside a GC. + Zone *zone = cx->zone(); + MOZ_ASSERT(!cx->runtime()->isHeapBusy(), "allocating while under GC"); + + AutoMaybeStartBackgroundAllocation maybeStartBGAlloc; + return cx->arenas()->allocateFromArena(zone, thingKind, CheckThresholds, maybeStartBGAlloc); +} + +/* static */ TenuredCell* +GCRuntime::refillFreeListOffMainThread(ExclusiveContext* cx, AllocKind thingKind) +{ + // A GC may be happening on the main thread, but zones used by exclusive + // contexts are never collected. + Zone* zone = cx->zone(); + MOZ_ASSERT(!zone->wasGCStarted()); + + AutoMaybeStartBackgroundAllocation maybeStartBGAlloc; + return cx->arenas()->allocateFromArena(zone, thingKind, CheckThresholds, maybeStartBGAlloc); +} + +/* static */ TenuredCell* +GCRuntime::refillFreeListInGC(Zone* zone, AllocKind thingKind) +{ + /* + * Called by compacting GC to refill a free list while we are in a GC. + */ + + zone->arenas.checkEmptyFreeList(thingKind); + mozilla::DebugOnly<JSRuntime*> rt = zone->runtimeFromMainThread(); + MOZ_ASSERT(rt->isHeapCollecting()); + MOZ_ASSERT_IF(!rt->isHeapMinorCollecting(), !rt->gc.isBackgroundSweeping()); + + AutoMaybeStartBackgroundAllocation maybeStartBackgroundAllocation; + return zone->arenas.allocateFromArena(zone, thingKind, DontCheckThresholds, + maybeStartBackgroundAllocation); +} + +TenuredCell* +ArenaLists::allocateFromArena(JS::Zone* zone, AllocKind thingKind, + ShouldCheckThresholds checkThresholds, + AutoMaybeStartBackgroundAllocation& maybeStartBGAlloc) +{ + JSRuntime* rt = zone->runtimeFromAnyThread(); + + mozilla::Maybe<AutoLockGC> maybeLock; + + // See if we can proceed without taking the GC lock. + if (backgroundFinalizeState[thingKind] != BFS_DONE) + maybeLock.emplace(rt); + + ArenaList& al = arenaLists[thingKind]; + Arena* arena = al.takeNextArena(); + if (arena) { + // Empty arenas should be immediately freed. + MOZ_ASSERT(!arena->isEmpty()); + + return allocateFromArenaInner(zone, arena, thingKind); + } + + // Parallel threads have their own ArenaLists, but chunks are shared; + // if we haven't already, take the GC lock now to avoid racing. + if (maybeLock.isNothing()) + maybeLock.emplace(rt); + + Chunk* chunk = rt->gc.pickChunk(maybeLock.ref(), maybeStartBGAlloc); + if (!chunk) + return nullptr; + + // Although our chunk should definitely have enough space for another arena, + // there are other valid reasons why Chunk::allocateArena() may fail. + arena = rt->gc.allocateArena(chunk, zone, thingKind, checkThresholds, maybeLock.ref()); + if (!arena) + return nullptr; + + MOZ_ASSERT(al.isCursorAtEnd()); + al.insertBeforeCursor(arena); + + return allocateFromArenaInner(zone, arena, thingKind); +} + +inline TenuredCell* +ArenaLists::allocateFromArenaInner(JS::Zone* zone, Arena* arena, AllocKind kind) +{ + size_t thingSize = Arena::thingSize(kind); + + freeLists[kind] = arena->getFirstFreeSpan(); + + if (MOZ_UNLIKELY(zone->wasGCStarted())) + zone->runtimeFromAnyThread()->gc.arenaAllocatedDuringGC(zone, arena); + TenuredCell* thing = freeLists[kind]->allocate(thingSize); + MOZ_ASSERT(thing); // This allocation is infallible. + return thing; +} + +void +GCRuntime::arenaAllocatedDuringGC(JS::Zone* zone, Arena* arena) +{ + if (zone->needsIncrementalBarrier()) { + arena->allocatedDuringIncremental = true; + marker.delayMarkingArena(arena); + } else if (zone->isGCSweeping()) { + arena->setNextAllocDuringSweep(arenasAllocatedDuringSweep); + arenasAllocatedDuringSweep = arena; + } +} + + +// /////////// Chunk -> Arena Allocator ////////////////////////////////////// + +bool +GCRuntime::wantBackgroundAllocation(const AutoLockGC& lock) const +{ + // To minimize memory waste, we do not want to run the background chunk + // allocation if we already have some empty chunks or when the runtime has + // a small heap size (and therefore likely has a small growth rate). + return allocTask.enabled() && + emptyChunks(lock).count() < tunables.minEmptyChunkCount(lock) && + (fullChunks(lock).count() + availableChunks(lock).count()) >= 4; +} + +Arena* +GCRuntime::allocateArena(Chunk* chunk, Zone* zone, AllocKind thingKind, + ShouldCheckThresholds checkThresholds, const AutoLockGC& lock) +{ + MOZ_ASSERT(chunk->hasAvailableArenas()); + + // Fail the allocation if we are over our heap size limits. + if (checkThresholds && usage.gcBytes() >= tunables.gcMaxBytes()) + return nullptr; + + Arena* arena = chunk->allocateArena(rt, zone, thingKind, lock); + zone->usage.addGCArena(); + + // Trigger an incremental slice if needed. + if (checkThresholds) + maybeAllocTriggerZoneGC(zone, lock); + + return arena; +} + +Arena* +Chunk::allocateArena(JSRuntime* rt, Zone* zone, AllocKind thingKind, const AutoLockGC& lock) +{ + Arena* arena = info.numArenasFreeCommitted > 0 + ? fetchNextFreeArena(rt) + : fetchNextDecommittedArena(); + arena->init(zone, thingKind); + updateChunkListAfterAlloc(rt, lock); + return arena; +} + +inline void +GCRuntime::updateOnFreeArenaAlloc(const ChunkInfo& info) +{ + MOZ_ASSERT(info.numArenasFreeCommitted <= numArenasFreeCommitted); + --numArenasFreeCommitted; +} + +Arena* +Chunk::fetchNextFreeArena(JSRuntime* rt) +{ + MOZ_ASSERT(info.numArenasFreeCommitted > 0); + MOZ_ASSERT(info.numArenasFreeCommitted <= info.numArenasFree); + + Arena* arena = info.freeArenasHead; + info.freeArenasHead = arena->next; + --info.numArenasFreeCommitted; + --info.numArenasFree; + rt->gc.updateOnFreeArenaAlloc(info); + + return arena; +} + +Arena* +Chunk::fetchNextDecommittedArena() +{ + MOZ_ASSERT(info.numArenasFreeCommitted == 0); + MOZ_ASSERT(info.numArenasFree > 0); + + unsigned offset = findDecommittedArenaOffset(); + info.lastDecommittedArenaOffset = offset + 1; + --info.numArenasFree; + decommittedArenas.unset(offset); + + Arena* arena = &arenas[offset]; + MarkPagesInUse(arena, ArenaSize); + arena->setAsNotAllocated(); + + return arena; +} + +/* + * Search for and return the next decommitted Arena. Our goal is to keep + * lastDecommittedArenaOffset "close" to a free arena. We do this by setting + * it to the most recently freed arena when we free, and forcing it to + * the last alloc + 1 when we allocate. + */ +uint32_t +Chunk::findDecommittedArenaOffset() +{ + /* Note: lastFreeArenaOffset can be past the end of the list. */ + for (unsigned i = info.lastDecommittedArenaOffset; i < ArenasPerChunk; i++) { + if (decommittedArenas.get(i)) + return i; + } + for (unsigned i = 0; i < info.lastDecommittedArenaOffset; i++) { + if (decommittedArenas.get(i)) + return i; + } + MOZ_CRASH("No decommitted arenas found."); +} + + +// /////////// System -> Chunk Allocator ///////////////////////////////////// + +Chunk* +GCRuntime::getOrAllocChunk(const AutoLockGC& lock, + AutoMaybeStartBackgroundAllocation& maybeStartBackgroundAllocation) +{ + Chunk* chunk = emptyChunks(lock).pop(); + if (!chunk) { + chunk = Chunk::allocate(rt); + if (!chunk) + return nullptr; + MOZ_ASSERT(chunk->info.numArenasFreeCommitted == 0); + } + + if (wantBackgroundAllocation(lock)) + maybeStartBackgroundAllocation.tryToStartBackgroundAllocation(rt->gc); + + return chunk; +} + +void +GCRuntime::recycleChunk(Chunk* chunk, const AutoLockGC& lock) +{ + emptyChunks(lock).push(chunk); +} + +Chunk* +GCRuntime::pickChunk(const AutoLockGC& lock, + AutoMaybeStartBackgroundAllocation& maybeStartBackgroundAllocation) +{ + if (availableChunks(lock).count()) + return availableChunks(lock).head(); + + Chunk* chunk = getOrAllocChunk(lock, maybeStartBackgroundAllocation); + if (!chunk) + return nullptr; + + chunk->init(rt); + MOZ_ASSERT(chunk->info.numArenasFreeCommitted == 0); + MOZ_ASSERT(chunk->unused()); + MOZ_ASSERT(!fullChunks(lock).contains(chunk)); + MOZ_ASSERT(!availableChunks(lock).contains(chunk)); + + chunkAllocationSinceLastGC = true; + + availableChunks(lock).push(chunk); + + return chunk; +} + +BackgroundAllocTask::BackgroundAllocTask(JSRuntime* rt, ChunkPool& pool) + : runtime(rt), + chunkPool_(pool), + enabled_(CanUseExtraThreads() && GetCPUCount() >= 2) +{ +} + +/* virtual */ void +BackgroundAllocTask::run() +{ + TraceLoggerThread* logger = TraceLoggerForCurrentThread(); + AutoTraceLog logAllocation(logger, TraceLogger_GCAllocation); + + AutoLockGC lock(runtime); + while (!cancel_ && runtime->gc.wantBackgroundAllocation(lock)) { + Chunk* chunk; + { + AutoUnlockGC unlock(lock); + chunk = Chunk::allocate(runtime); + if (!chunk) + break; + chunk->init(runtime); + } + chunkPool_.push(chunk); + } +} + +/* static */ Chunk* +Chunk::allocate(JSRuntime* rt) +{ + Chunk* chunk = static_cast<Chunk*>(MapAlignedPages(ChunkSize, ChunkSize)); + if (!chunk) + return nullptr; + rt->gc.stats.count(gcstats::STAT_NEW_CHUNK); + return chunk; +} + +void +Chunk::init(JSRuntime* rt) +{ + JS_POISON(this, JS_FRESH_TENURED_PATTERN, ChunkSize); + + /* + * We clear the bitmap to guard against JS::GCThingIsMarkedGray being called + * on uninitialized data, which would happen before the first GC cycle. + */ + bitmap.clear(); + + /* + * Decommit the arenas. We do this after poisoning so that if the OS does + * not have to recycle the pages, we still get the benefit of poisoning. + */ + decommitAllArenas(rt); + + /* Initialize the chunk info. */ + info.init(); + new (&trailer) ChunkTrailer(rt); + + /* The rest of info fields are initialized in pickChunk. */ +} + +void Chunk::decommitAllArenas(JSRuntime* rt) +{ + decommittedArenas.clear(true); + MarkPagesUnused(&arenas[0], ArenasPerChunk * ArenaSize); + + info.freeArenasHead = nullptr; + info.lastDecommittedArenaOffset = 0; + info.numArenasFree = ArenasPerChunk; + info.numArenasFreeCommitted = 0; +} diff --git a/js/src/gc/Allocator.h b/js/src/gc/Allocator.h new file mode 100644 index 000000000..fd530cadf --- /dev/null +++ b/js/src/gc/Allocator.h @@ -0,0 +1,35 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_Allocator_h +#define gc_Allocator_h + +#include "gc/Heap.h" +#include "js/RootingAPI.h" + +namespace js { +struct Class; + +// Allocate a new GC thing. After a successful allocation the caller must +// fully initialize the thing before calling any function that can potentially +// trigger GC. This will ensure that GC tracing never sees junk values stored +// in the partially initialized thing. +// +// Note that JSObject allocation must use the longer signature below that +// includes slot, heap, and finalizer information in support of various +// object-specific optimizations. +template <typename T, AllowGC allowGC = CanGC> +T* +Allocate(ExclusiveContext* cx); + +template <typename, AllowGC allowGC = CanGC> +JSObject* +Allocate(ExclusiveContext* cx, gc::AllocKind kind, size_t nDynamicSlots, gc::InitialHeap heap, + const Class* clasp); + +} // namespace js + +#endif // gc_Allocator_h diff --git a/js/src/gc/Barrier.cpp b/js/src/gc/Barrier.cpp new file mode 100644 index 000000000..f19f6f046 --- /dev/null +++ b/js/src/gc/Barrier.cpp @@ -0,0 +1,210 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/Barrier.h" + +#include "jscompartment.h" +#include "jsobj.h" + +#include "builtin/TypedObject.h" +#include "gc/Policy.h" +#include "gc/Zone.h" +#include "js/HashTable.h" +#include "js/Value.h" +#include "vm/EnvironmentObject.h" +#include "vm/SharedArrayObject.h" +#include "vm/Symbol.h" +#include "wasm/WasmJS.h" + +namespace js { + +bool +RuntimeFromMainThreadIsHeapMajorCollecting(JS::shadow::Zone* shadowZone) +{ + return shadowZone->runtimeFromMainThread()->isHeapMajorCollecting(); +} + +#ifdef DEBUG + +bool +IsMarkedBlack(NativeObject* obj) +{ + // Note: we assume conservatively that Nursery things will be live. + if (!obj->isTenured()) + return true; + + gc::TenuredCell& tenured = obj->asTenured(); + return (tenured.isMarked(gc::BLACK) && !tenured.isMarked(gc::GRAY)) || + tenured.arena()->allocatedDuringIncremental; +} + +bool +HeapSlot::preconditionForSet(NativeObject* owner, Kind kind, uint32_t slot) const +{ + return kind == Slot + ? &owner->getSlotRef(slot) == this + : &owner->getDenseElement(slot) == (const Value*)this; +} + +bool +HeapSlot::preconditionForWriteBarrierPost(NativeObject* obj, Kind kind, uint32_t slot, + const Value& target) const +{ + bool isCorrectSlot = kind == Slot + ? obj->getSlotAddressUnchecked(slot)->get() == target + : static_cast<HeapSlot*>(obj->getDenseElements() + slot)->get() == target; + bool isBlackToGray = target.isMarkable() && + IsMarkedBlack(obj) && JS::GCThingIsMarkedGray(JS::GCCellPtr(target)); + return isCorrectSlot && !isBlackToGray; +} + +bool +CurrentThreadIsIonCompiling() +{ + return TlsPerThreadData.get()->ionCompiling; +} + +bool +CurrentThreadIsIonCompilingSafeForMinorGC() +{ + return TlsPerThreadData.get()->ionCompilingSafeForMinorGC; +} + +bool +CurrentThreadIsGCSweeping() +{ + return TlsPerThreadData.get()->gcSweeping; +} + +#endif // DEBUG + +template <typename S> +template <typename T> +void +ReadBarrierFunctor<S>::operator()(T* t) +{ + InternalBarrierMethods<T*>::readBarrier(t); +} + +// All GC things may be held in a Value, either publicly or as a private GC +// thing. +#define JS_EXPAND_DEF(name, type, _) \ +template void ReadBarrierFunctor<JS::Value>::operator()<type>(type*); +JS_FOR_EACH_TRACEKIND(JS_EXPAND_DEF); +#undef JS_EXPAND_DEF + +template <typename S> +template <typename T> +void +PreBarrierFunctor<S>::operator()(T* t) +{ + InternalBarrierMethods<T*>::preBarrier(t); +} + +// All GC things may be held in a Value, either publicly or as a private GC +// thing. +#define JS_EXPAND_DEF(name, type, _) \ +template void PreBarrierFunctor<JS::Value>::operator()<type>(type*); +JS_FOR_EACH_TRACEKIND(JS_EXPAND_DEF); +#undef JS_EXPAND_DEF + +template void PreBarrierFunctor<jsid>::operator()<JS::Symbol>(JS::Symbol*); +template void PreBarrierFunctor<jsid>::operator()<JSString>(JSString*); + +template <typename T> +/* static */ bool +MovableCellHasher<T>::hasHash(const Lookup& l) +{ + if (!l) + return true; + + return l->zoneFromAnyThread()->hasUniqueId(l); +} + +template <typename T> +/* static */ bool +MovableCellHasher<T>::ensureHash(const Lookup& l) +{ + if (!l) + return true; + + uint64_t unusedId; + return l->zoneFromAnyThread()->getUniqueId(l, &unusedId); +} + +template <typename T> +/* static */ HashNumber +MovableCellHasher<T>::hash(const Lookup& l) +{ + if (!l) + return 0; + + // We have to access the zone from-any-thread here: a worker thread may be + // cloning a self-hosted object from the main-thread-runtime-owned self- + // hosting zone into the off-main-thread runtime. The zone's uid lock will + // protect against multiple workers doing this simultaneously. + MOZ_ASSERT(CurrentThreadCanAccessZone(l->zoneFromAnyThread()) || + l->zoneFromAnyThread()->isSelfHostingZone()); + + return l->zoneFromAnyThread()->getHashCodeInfallible(l); +} + +template <typename T> +/* static */ bool +MovableCellHasher<T>::match(const Key& k, const Lookup& l) +{ + // Return true if both are null or false if only one is null. + if (!k) + return !l; + if (!l) + return false; + + MOZ_ASSERT(k); + MOZ_ASSERT(l); + MOZ_ASSERT(CurrentThreadCanAccessZone(l->zoneFromAnyThread()) || + l->zoneFromAnyThread()->isSelfHostingZone()); + + Zone* zone = k->zoneFromAnyThread(); + if (zone != l->zoneFromAnyThread()) + return false; + MOZ_ASSERT(zone->hasUniqueId(k)); + MOZ_ASSERT(zone->hasUniqueId(l)); + + // Since both already have a uid (from hash), the get is infallible. + return zone->getUniqueIdInfallible(k) == zone->getUniqueIdInfallible(l); +} + +#ifdef JS_BROKEN_GCC_ATTRIBUTE_WARNING +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wattributes" +#endif // JS_BROKEN_GCC_ATTRIBUTE_WARNING + +template struct JS_PUBLIC_API(MovableCellHasher<JSObject*>); +template struct JS_PUBLIC_API(MovableCellHasher<GlobalObject*>); +template struct JS_PUBLIC_API(MovableCellHasher<SavedFrame*>); +template struct JS_PUBLIC_API(MovableCellHasher<EnvironmentObject*>); +template struct JS_PUBLIC_API(MovableCellHasher<WasmInstanceObject*>); +template struct JS_PUBLIC_API(MovableCellHasher<JSScript*>); + +#ifdef JS_BROKEN_GCC_ATTRIBUTE_WARNING +#pragma GCC diagnostic pop +#endif // JS_BROKEN_GCC_ATTRIBUTE_WARNING + +} // namespace js + +JS_PUBLIC_API(void) +JS::HeapObjectPostBarrier(JSObject** objp, JSObject* prev, JSObject* next) +{ + MOZ_ASSERT(objp); + js::InternalBarrierMethods<JSObject*>::postBarrier(objp, prev, next); +} + +JS_PUBLIC_API(void) +JS::HeapValuePostBarrier(JS::Value* valuep, const Value& prev, const Value& next) +{ + MOZ_ASSERT(valuep); + js::InternalBarrierMethods<JS::Value>::postBarrier(valuep, prev, next); +} diff --git a/js/src/gc/Barrier.h b/js/src/gc/Barrier.h new file mode 100644 index 000000000..950c96314 --- /dev/null +++ b/js/src/gc/Barrier.h @@ -0,0 +1,968 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_Barrier_h +#define gc_Barrier_h + +#include "NamespaceImports.h" + +#include "gc/Heap.h" +#include "gc/StoreBuffer.h" +#include "js/HeapAPI.h" +#include "js/Id.h" +#include "js/RootingAPI.h" +#include "js/Value.h" + +/* + * A write barrier is a mechanism used by incremental or generation GCs to + * ensure that every value that needs to be marked is marked. In general, the + * write barrier should be invoked whenever a write can cause the set of things + * traced through by the GC to change. This includes: + * - writes to object properties + * - writes to array slots + * - writes to fields like JSObject::shape_ that we trace through + * - writes to fields in private data + * - writes to non-markable fields like JSObject::private that point to + * markable data + * The last category is the trickiest. Even though the private pointers does not + * point to a GC thing, changing the private pointer may change the set of + * objects that are traced by the GC. Therefore it needs a write barrier. + * + * Every barriered write should have the following form: + * <pre-barrier> + * obj->field = value; // do the actual write + * <post-barrier> + * The pre-barrier is used for incremental GC and the post-barrier is for + * generational GC. + * + * PRE-BARRIER + * + * To understand the pre-barrier, let's consider how incremental GC works. The + * GC itself is divided into "slices". Between each slice, JS code is allowed to + * run. Each slice should be short so that the user doesn't notice the + * interruptions. In our GC, the structure of the slices is as follows: + * + * 1. ... JS work, which leads to a request to do GC ... + * 2. [first GC slice, which performs all root marking and possibly more marking] + * 3. ... more JS work is allowed to run ... + * 4. [GC mark slice, which runs entirely in drainMarkStack] + * 5. ... more JS work ... + * 6. [GC mark slice, which runs entirely in drainMarkStack] + * 7. ... more JS work ... + * 8. [GC marking finishes; sweeping done non-incrementally; GC is done] + * 9. ... JS continues uninterrupted now that GC is finishes ... + * + * Of course, there may be a different number of slices depending on how much + * marking is to be done. + * + * The danger inherent in this scheme is that the JS code in steps 3, 5, and 7 + * might change the heap in a way that causes the GC to collect an object that + * is actually reachable. The write barrier prevents this from happening. We use + * a variant of incremental GC called "snapshot at the beginning." This approach + * guarantees the invariant that if an object is reachable in step 2, then we + * will mark it eventually. The name comes from the idea that we take a + * theoretical "snapshot" of all reachable objects in step 2; all objects in + * that snapshot should eventually be marked. (Note that the write barrier + * verifier code takes an actual snapshot.) + * + * The basic correctness invariant of a snapshot-at-the-beginning collector is + * that any object reachable at the end of the GC (step 9) must either: + * (1) have been reachable at the beginning (step 2) and thus in the snapshot + * (2) or must have been newly allocated, in steps 3, 5, or 7. + * To deal with case (2), any objects allocated during an incremental GC are + * automatically marked black. + * + * This strategy is actually somewhat conservative: if an object becomes + * unreachable between steps 2 and 8, it would be safe to collect it. We won't, + * mainly for simplicity. (Also, note that the snapshot is entirely + * theoretical. We don't actually do anything special in step 2 that we wouldn't + * do in a non-incremental GC. + * + * It's the pre-barrier's job to maintain the snapshot invariant. Consider the + * write "obj->field = value". Let the prior value of obj->field be + * value0. Since it's possible that value0 may have been what obj->field + * contained in step 2, when the snapshot was taken, the barrier marks + * value0. Note that it only does this if we're in the middle of an incremental + * GC. Since this is rare, the cost of the write barrier is usually just an + * extra branch. + * + * In practice, we implement the pre-barrier differently based on the type of + * value0. E.g., see JSObject::writeBarrierPre, which is used if obj->field is + * a JSObject*. It takes value0 as a parameter. + * + * READ-BARRIER + * + * Incremental GC requires that weak pointers have read barriers. The problem + * happens when, during an incremental GC, some code reads a weak pointer and + * writes it somewhere on the heap that has been marked black in a previous + * slice. Since the weak pointer will not otherwise be marked and will be swept + * and finalized in the last slice, this will leave the pointer just written + * dangling after the GC. To solve this, we immediately mark black all weak + * pointers that get read between slices so that it is safe to store them in an + * already marked part of the heap, e.g. in Rooted. + * + * POST-BARRIER + * + * For generational GC, we want to be able to quickly collect the nursery in a + * minor collection. Part of the way this is achieved is to only mark the + * nursery itself; tenured things, which may form the majority of the heap, are + * not traced through or marked. This leads to the problem of what to do about + * tenured objects that have pointers into the nursery: if such things are not + * marked, they may be discarded while there are still live objects which + * reference them. The solution is to maintain information about these pointers, + * and mark their targets when we start a minor collection. + * + * The pointers can be thought of as edges in object graph, and the set of edges + * from the tenured generation into the nursery is know as the remembered set. + * Post barriers are used to track this remembered set. + * + * Whenever a slot which could contain such a pointer is written, we use a write + * barrier to check if the edge created is in the remembered set, and if so we + * insert it into the store buffer, which is the collector's representation of + * the remembered set. This means than when we come to do a minor collection we + * can examine the contents of the store buffer and mark any edge targets that + * are in the nursery. + * + * IMPLEMENTATION DETAILS + * + * Since it would be awkward to change every write to memory into a function + * call, this file contains a bunch of C++ classes and templates that use + * operator overloading to take care of barriers automatically. In many cases, + * all that's necessary to make some field be barriered is to replace + * Type* field; + * with + * GCPtr<Type> field; + * + * One additional note: not all object writes need to be pre-barriered. Writes + * to newly allocated objects do not need a pre-barrier. In these cases, we use + * the "obj->field.init(value)" method instead of "obj->field = value". We use + * the init naming idiom in many places to signify that a field is being + * assigned for the first time. + * + * This file implements four classes, illustrated here: + * + * BarrieredBase base class of all barriers + * | | + * | WriteBarrieredBase base class which provides common write operations + * | | | | | + * | | | | PreBarriered provides pre-barriers only + * | | | | + * | | | GCPtr provides pre- and post-barriers + * | | | + * | | HeapPtr provides pre- and post-barriers; is relocatable + * | | and deletable for use inside C++ managed memory + * | | + * | HeapSlot similar to GCPtr, but tailored to slots storage + * | + * ReadBarrieredBase base class which provides common read operations + * | + * ReadBarriered provides read barriers only + * + * + * The implementation of the barrier logic is implemented on T::writeBarrier.*, + * via: + * + * WriteBarrieredBase<T>::pre + * -> InternalBarrierMethods<T*>::preBarrier + * -> T::writeBarrierPre + * -> InternalBarrierMethods<Value>::preBarrier + * -> InternalBarrierMethods<jsid>::preBarrier + * -> InternalBarrierMethods<T*>::preBarrier + * -> T::writeBarrierPre + * + * GCPtr<T>::post and HeapPtr<T>::post + * -> InternalBarrierMethods<T*>::postBarrier + * -> T::writeBarrierPost + * -> InternalBarrierMethods<Value>::postBarrier + * -> StoreBuffer::put + * + * These classes are designed to be used by the internals of the JS engine. + * Barriers designed to be used externally are provided in js/RootingAPI.h. + * These external barriers call into the same post-barrier implementations at + * InternalBarrierMethods<T>::post via an indirect call to Heap(.+)Barrier. + * + * These clases are designed to be used to wrap GC thing pointers or values that + * act like them (i.e. JS::Value and jsid). It is possible to use them for + * other types by supplying the necessary barrier implementations but this + * is not usually necessary and should be done with caution. + */ + +class JSAtom; +struct JSCompartment; +class JSFlatString; +class JSLinearString; + +namespace JS { +class Symbol; +} // namespace JS + +namespace js { + +class AccessorShape; +class ArrayObject; +class ArgumentsObject; +class ArrayBufferObjectMaybeShared; +class ArrayBufferObject; +class ArrayBufferViewObject; +class SharedArrayBufferObject; +class BaseShape; +class DebugEnvironmentProxy; +class GlobalObject; +class LazyScript; +class ModuleObject; +class ModuleEnvironmentObject; +class ModuleNamespaceObject; +class NativeObject; +class PlainObject; +class PropertyName; +class SavedFrame; +class EnvironmentObject; +class ScriptSourceObject; +class Shape; +class UnownedBaseShape; +class ObjectGroup; + +namespace jit { +class JitCode; +} // namespace jit + +#ifdef DEBUG +// Barriers can't be triggered during backend Ion compilation, which may run on +// a helper thread. +bool +CurrentThreadIsIonCompiling(); + +bool +CurrentThreadIsIonCompilingSafeForMinorGC(); + +bool +CurrentThreadIsGCSweeping(); + +bool +IsMarkedBlack(NativeObject* obj); +#endif + +namespace gc { + +// Marking.h depends on these barrier definitions, so we need a separate +// entry point for marking to implement the pre-barrier. +void MarkValueForBarrier(JSTracer* trc, Value* v, const char* name); +void MarkIdForBarrier(JSTracer* trc, jsid* idp, const char* name); + +} // namespace gc + +template <typename T> +struct InternalBarrierMethods {}; + +template <typename T> +struct InternalBarrierMethods<T*> +{ + static bool isMarkable(T* v) { return v != nullptr; } + + static bool isMarkableTaggedPointer(T* v) { return !IsNullTaggedPointer(v); } + + static void preBarrier(T* v) { T::writeBarrierPre(v); } + + static void postBarrier(T** vp, T* prev, T* next) { T::writeBarrierPost(vp, prev, next); } + + static void readBarrier(T* v) { T::readBarrier(v); } +}; + +template <typename S> struct PreBarrierFunctor : public VoidDefaultAdaptor<S> { + template <typename T> void operator()(T* t); +}; + +template <typename S> struct ReadBarrierFunctor : public VoidDefaultAdaptor<S> { + template <typename T> void operator()(T* t); +}; + +template <> +struct InternalBarrierMethods<Value> +{ + static bool isMarkable(const Value& v) { return v.isMarkable(); } + static bool isMarkableTaggedPointer(const Value& v) { return isMarkable(v); } + + static void preBarrier(const Value& v) { + DispatchTyped(PreBarrierFunctor<Value>(), v); + } + + static void postBarrier(Value* vp, const Value& prev, const Value& next) { + MOZ_ASSERT(!CurrentThreadIsIonCompiling()); + MOZ_ASSERT(vp); + + // If the target needs an entry, add it. + js::gc::StoreBuffer* sb; + if (next.isObject() && (sb = reinterpret_cast<gc::Cell*>(&next.toObject())->storeBuffer())) { + // If we know that the prev has already inserted an entry, we can + // skip doing the lookup to add the new entry. Note that we cannot + // safely assert the presence of the entry because it may have been + // added via a different store buffer. + if (prev.isObject() && reinterpret_cast<gc::Cell*>(&prev.toObject())->storeBuffer()) + return; + sb->putValue(vp); + return; + } + // Remove the prev entry if the new value does not need it. + if (prev.isObject() && (sb = reinterpret_cast<gc::Cell*>(&prev.toObject())->storeBuffer())) + sb->unputValue(vp); + } + + static void readBarrier(const Value& v) { + DispatchTyped(ReadBarrierFunctor<Value>(), v); + } +}; + +template <> +struct InternalBarrierMethods<jsid> +{ + static bool isMarkable(jsid id) { return JSID_IS_GCTHING(id); } + static bool isMarkableTaggedPointer(jsid id) { return isMarkable(id); } + + static void preBarrier(jsid id) { DispatchTyped(PreBarrierFunctor<jsid>(), id); } + static void postBarrier(jsid* idp, jsid prev, jsid next) {} +}; + +// Barrier classes can use Mixins to add methods to a set of barrier +// instantiations, to make the barriered thing look and feel more like the +// thing itself. +template <typename T> +class BarrieredBaseMixins {}; + +// Base class of all barrier types. +// +// This is marked non-memmovable since post barriers added by derived classes +// can add pointers to class instances to the store buffer. +template <typename T> +class MOZ_NON_MEMMOVABLE BarrieredBase : public BarrieredBaseMixins<T> +{ + protected: + // BarrieredBase is not directly instantiable. + explicit BarrieredBase(const T& v) : value(v) {} + + // Storage for all barrier classes. |value| must be a GC thing reference + // type: either a direct pointer to a GC thing or a supported tagged + // pointer that can reference GC things, such as JS::Value or jsid. Nested + // barrier types are NOT supported. See assertTypeConstraints. + T value; + + public: + // Note: this is public because C++ cannot friend to a specific template instantiation. + // Friending to the generic template leads to a number of unintended consequences, including + // template resolution ambiguity and a circular dependency with Tracing.h. + T* unsafeUnbarrieredForTracing() { return &value; } +}; + +// Base class for barriered pointer types that intercept only writes. +template <class T> +class WriteBarrieredBase : public BarrieredBase<T> +{ + protected: + // WriteBarrieredBase is not directly instantiable. + explicit WriteBarrieredBase(const T& v) : BarrieredBase<T>(v) {} + + public: + DECLARE_POINTER_COMPARISON_OPS(T); + DECLARE_POINTER_CONSTREF_OPS(T); + + // Use this if the automatic coercion to T isn't working. + const T& get() const { return this->value; } + + // Use this if you want to change the value without invoking barriers. + // Obviously this is dangerous unless you know the barrier is not needed. + void unsafeSet(const T& v) { this->value = v; } + + // For users who need to manually barrier the raw types. + static void writeBarrierPre(const T& v) { InternalBarrierMethods<T>::preBarrier(v); } + + protected: + void pre() { InternalBarrierMethods<T>::preBarrier(this->value); } + void post(const T& prev, const T& next) { + InternalBarrierMethods<T>::postBarrier(&this->value, prev, next); + } +}; + +/* + * PreBarriered only automatically handles pre-barriers. Post-barriers must be + * manually implemented when using this class. GCPtr and HeapPtr should be used + * in all cases that do not require explicit low-level control of moving + * behavior, e.g. for HashMap keys. + */ +template <class T> +class PreBarriered : public WriteBarrieredBase<T> +{ + public: + PreBarriered() : WriteBarrieredBase<T>(JS::GCPolicy<T>::initial()) {} + /* + * Allow implicit construction for use in generic contexts, such as + * DebuggerWeakMap::markKeys. + */ + MOZ_IMPLICIT PreBarriered(const T& v) : WriteBarrieredBase<T>(v) {} + explicit PreBarriered(const PreBarriered<T>& v) : WriteBarrieredBase<T>(v.value) {} + ~PreBarriered() { this->pre(); } + + void init(const T& v) { + this->value = v; + } + + /* Use to set the pointer to nullptr. */ + void clear() { + this->pre(); + this->value = nullptr; + } + + DECLARE_POINTER_ASSIGN_OPS(PreBarriered, T); + + private: + void set(const T& v) { + this->pre(); + this->value = v; + } +}; + +/* + * A pre- and post-barriered heap pointer, for use inside the JS engine. + * + * It must only be stored in memory that has GC lifetime. GCPtr must not be + * used in contexts where it may be implicitly moved or deleted, e.g. most + * containers. + * + * The post-barriers implemented by this class are faster than those + * implemented by js::HeapPtr<T> or JS::Heap<T> at the cost of not + * automatically handling deletion or movement. + */ +template <class T> +class GCPtr : public WriteBarrieredBase<T> +{ + public: + GCPtr() : WriteBarrieredBase<T>(JS::GCPolicy<T>::initial()) {} + explicit GCPtr(const T& v) : WriteBarrieredBase<T>(v) { + this->post(JS::GCPolicy<T>::initial(), v); + } + explicit GCPtr(const GCPtr<T>& v) : WriteBarrieredBase<T>(v) { + this->post(JS::GCPolicy<T>::initial(), v); + } +#ifdef DEBUG + ~GCPtr() { + // No prebarrier necessary as this only happens when we are sweeping or + // after we have just collected the nursery. Note that the wrapped + // pointer may already have been freed by this point. + MOZ_ASSERT(CurrentThreadIsGCSweeping()); + Poison(this, JS_FREED_HEAP_PTR_PATTERN, sizeof(*this)); + } +#endif + + void init(const T& v) { + this->value = v; + this->post(JS::GCPolicy<T>::initial(), v); + } + + DECLARE_POINTER_ASSIGN_OPS(GCPtr, T); + + T unbarrieredGet() const { + return this->value; + } + + private: + void set(const T& v) { + this->pre(); + T tmp = this->value; + this->value = v; + this->post(tmp, this->value); + } + + /* + * Unlike HeapPtr<T>, GCPtr<T> must be managed with GC lifetimes. + * Specifically, the memory used by the pointer itself must be live until + * at least the next minor GC. For that reason, move semantics are invalid + * and are deleted here. Please note that not all containers support move + * semantics, so this does not completely prevent invalid uses. + */ + GCPtr(GCPtr<T>&&) = delete; + GCPtr<T>& operator=(GCPtr<T>&&) = delete; +}; + +/* + * A pre- and post-barriered heap pointer, for use inside the JS engine. These + * heap pointers can be stored in C++ containers like GCVector and GCHashMap. + * + * The GC sometimes keeps pointers to pointers to GC things --- for example, to + * track references into the nursery. However, C++ containers like GCVector and + * GCHashMap usually reserve the right to relocate their elements any time + * they're modified, invalidating all pointers to the elements. HeapPtr + * has a move constructor which knows how to keep the GC up to date if it is + * moved to a new location. + * + * However, because of this additional communication with the GC, HeapPtr + * is somewhat slower, so it should only be used in contexts where this ability + * is necessary. + * + * Obviously, JSObjects, JSStrings, and the like get tenured and compacted, so + * whatever pointers they contain get relocated, in the sense used here. + * However, since the GC itself is moving those values, it takes care of its + * internal pointers to those pointers itself. HeapPtr is only necessary + * when the relocation would otherwise occur without the GC's knowledge. + */ +template <class T> +class HeapPtr : public WriteBarrieredBase<T> +{ + public: + HeapPtr() : WriteBarrieredBase<T>(JS::GCPolicy<T>::initial()) {} + + // Implicitly adding barriers is a reasonable default. + MOZ_IMPLICIT HeapPtr(const T& v) : WriteBarrieredBase<T>(v) { + this->post(JS::GCPolicy<T>::initial(), this->value); + } + + /* + * For HeapPtr, move semantics are equivalent to copy semantics. In + * C++, a copy constructor taking const-ref is the way to get a single + * function that will be used for both lvalue and rvalue copies, so we can + * simply omit the rvalue variant. + */ + MOZ_IMPLICIT HeapPtr(const HeapPtr<T>& v) : WriteBarrieredBase<T>(v) { + this->post(JS::GCPolicy<T>::initial(), this->value); + } + + ~HeapPtr() { + this->pre(); + this->post(this->value, JS::GCPolicy<T>::initial()); + } + + void init(const T& v) { + this->value = v; + this->post(JS::GCPolicy<T>::initial(), this->value); + } + + DECLARE_POINTER_ASSIGN_OPS(HeapPtr, T); + + /* Make this friend so it can access pre() and post(). */ + template <class T1, class T2> + friend inline void + BarrieredSetPair(Zone* zone, + HeapPtr<T1*>& v1, T1* val1, + HeapPtr<T2*>& v2, T2* val2); + + protected: + void set(const T& v) { + this->pre(); + postBarrieredSet(v); + } + + void postBarrieredSet(const T& v) { + T tmp = this->value; + this->value = v; + this->post(tmp, this->value); + } +}; + +// Base class for barriered pointer types that intercept reads and writes. +template <typename T> +class ReadBarrieredBase : public BarrieredBase<T> +{ + protected: + // ReadBarrieredBase is not directly instantiable. + explicit ReadBarrieredBase(const T& v) : BarrieredBase<T>(v) {} + + protected: + void read() const { InternalBarrierMethods<T>::readBarrier(this->value); } + void post(const T& prev, const T& next) { + InternalBarrierMethods<T>::postBarrier(&this->value, prev, next); + } +}; + +// Incremental GC requires that weak pointers have read barriers. See the block +// comment at the top of Barrier.h for a complete discussion of why. +// +// Note that this class also has post-barriers, so is safe to use with nursery +// pointers. However, when used as a hashtable key, care must still be taken to +// insert manual post-barriers on the table for rekeying if the key is based in +// any way on the address of the object. +template <typename T> +class ReadBarriered : public ReadBarrieredBase<T> +{ + public: + ReadBarriered() : ReadBarrieredBase<T>(JS::GCPolicy<T>::initial()) {} + + // It is okay to add barriers implicitly. + MOZ_IMPLICIT ReadBarriered(const T& v) : ReadBarrieredBase<T>(v) { + this->post(JS::GCPolicy<T>::initial(), v); + } + + // Copy is creating a new edge, so we must read barrier the source edge. + explicit ReadBarriered(const ReadBarriered& v) : ReadBarrieredBase<T>(v) { + this->post(JS::GCPolicy<T>::initial(), v.get()); + } + + // Move retains the lifetime status of the source edge, so does not fire + // the read barrier of the defunct edge. + ReadBarriered(ReadBarriered&& v) + : ReadBarrieredBase<T>(mozilla::Move(v)) + { + this->post(JS::GCPolicy<T>::initial(), v.value); + } + + ~ReadBarriered() { + this->post(this->value, JS::GCPolicy<T>::initial()); + } + + ReadBarriered& operator=(const ReadBarriered& v) { + T prior = this->value; + this->value = v.value; + this->post(prior, v.value); + return *this; + } + + const T get() const { + if (!InternalBarrierMethods<T>::isMarkable(this->value)) + return JS::GCPolicy<T>::initial(); + this->read(); + return this->value; + } + + const T unbarrieredGet() const { + return this->value; + } + + explicit operator bool() const { + return bool(this->value); + } + + operator const T() const { return get(); } + + const T operator->() const { return get(); } + + T* unsafeGet() { return &this->value; } + T const* unsafeGet() const { return &this->value; } + + void set(const T& v) + { + T tmp = this->value; + this->value = v; + this->post(tmp, v); + } +}; + +// A WeakRef pointer does not hold its target live and is automatically nulled +// out when the GC discovers that it is not reachable from any other path. +template <typename T> +using WeakRef = ReadBarriered<T>; + +// Add Value operations to all Barrier types. Note, this must be defined before +// HeapSlot for HeapSlot's base to get these operations. +template <> +class BarrieredBaseMixins<JS::Value> : public ValueOperations<WriteBarrieredBase<JS::Value>> +{}; + +// A pre- and post-barriered Value that is specialized to be aware that it +// resides in a slots or elements vector. This allows it to be relocated in +// memory, but with substantially less overhead than a HeapPtr. +class HeapSlot : public WriteBarrieredBase<Value> +{ + public: + enum Kind { + Slot = 0, + Element = 1 + }; + + explicit HeapSlot() = delete; + + explicit HeapSlot(NativeObject* obj, Kind kind, uint32_t slot, const Value& v) + : WriteBarrieredBase<Value>(v) + { + post(obj, kind, slot, v); + } + + explicit HeapSlot(NativeObject* obj, Kind kind, uint32_t slot, const HeapSlot& s) + : WriteBarrieredBase<Value>(s.value) + { + post(obj, kind, slot, s); + } + + ~HeapSlot() { + pre(); + } + + void init(NativeObject* owner, Kind kind, uint32_t slot, const Value& v) { + value = v; + post(owner, kind, slot, v); + } + +#ifdef DEBUG + bool preconditionForSet(NativeObject* owner, Kind kind, uint32_t slot) const; + bool preconditionForWriteBarrierPost(NativeObject* obj, Kind kind, uint32_t slot, + const Value& target) const; +#endif + + void set(NativeObject* owner, Kind kind, uint32_t slot, const Value& v) { + MOZ_ASSERT(preconditionForSet(owner, kind, slot)); + pre(); + value = v; + post(owner, kind, slot, v); + } + + /* For users who need to manually barrier the raw types. */ + static void writeBarrierPost(NativeObject* owner, Kind kind, uint32_t slot, const Value& target) { + reinterpret_cast<HeapSlot*>(const_cast<Value*>(&target))->post(owner, kind, slot, target); + } + + private: + void post(NativeObject* owner, Kind kind, uint32_t slot, const Value& target) { + MOZ_ASSERT(preconditionForWriteBarrierPost(owner, kind, slot, target)); + if (this->value.isObject()) { + gc::Cell* cell = reinterpret_cast<gc::Cell*>(&this->value.toObject()); + if (cell->storeBuffer()) + cell->storeBuffer()->putSlot(owner, kind, slot, 1); + } + } +}; + +class HeapSlotArray +{ + HeapSlot* array; + + // Whether writes may be performed to the slots in this array. This helps + // to control how object elements which may be copy on write are used. +#ifdef DEBUG + bool allowWrite_; +#endif + + public: + explicit HeapSlotArray(HeapSlot* array, bool allowWrite) + : array(array) +#ifdef DEBUG + , allowWrite_(allowWrite) +#endif + {} + + operator const Value*() const { + JS_STATIC_ASSERT(sizeof(GCPtr<Value>) == sizeof(Value)); + JS_STATIC_ASSERT(sizeof(HeapSlot) == sizeof(Value)); + return reinterpret_cast<const Value*>(array); + } + operator HeapSlot*() const { MOZ_ASSERT(allowWrite()); return array; } + + HeapSlotArray operator +(int offset) const { return HeapSlotArray(array + offset, allowWrite()); } + HeapSlotArray operator +(uint32_t offset) const { return HeapSlotArray(array + offset, allowWrite()); } + + private: + bool allowWrite() const { +#ifdef DEBUG + return allowWrite_; +#else + return true; +#endif + } +}; + +/* + * This is a hack for RegExpStatics::updateFromMatch. It allows us to do two + * barriers with only one branch to check if we're in an incremental GC. + */ +template <class T1, class T2> +static inline void +BarrieredSetPair(Zone* zone, + HeapPtr<T1*>& v1, T1* val1, + HeapPtr<T2*>& v2, T2* val2) +{ + if (T1::needWriteBarrierPre(zone)) { + v1.pre(); + v2.pre(); + } + v1.postBarrieredSet(val1); + v2.postBarrieredSet(val2); +} + +/* + * ImmutableTenuredPtr is designed for one very narrow case: replacing + * immutable raw pointers to GC-managed things, implicitly converting to a + * handle type for ease of use. Pointers encapsulated by this type must: + * + * be immutable (no incremental write barriers), + * never point into the nursery (no generational write barriers), and + * be traced via MarkRuntime (we use fromMarkedLocation). + * + * In short: you *really* need to know what you're doing before you use this + * class! + */ +template <typename T> +class ImmutableTenuredPtr +{ + T value; + + public: + operator T() const { return value; } + T operator->() const { return value; } + + operator Handle<T>() const { + return Handle<T>::fromMarkedLocation(&value); + } + + void init(T ptr) { + MOZ_ASSERT(ptr->isTenured()); + value = ptr; + } + + T get() const { return value; } + const T* address() { return &value; } +}; + +template <typename T> +struct MovableCellHasher<PreBarriered<T>> +{ + using Key = PreBarriered<T>; + using Lookup = T; + + static bool hasHash(const Lookup& l) { return MovableCellHasher<T>::hasHash(l); } + static bool ensureHash(const Lookup& l) { return MovableCellHasher<T>::ensureHash(l); } + static HashNumber hash(const Lookup& l) { return MovableCellHasher<T>::hash(l); } + static bool match(const Key& k, const Lookup& l) { return MovableCellHasher<T>::match(k, l); } + static void rekey(Key& k, const Key& newKey) { k.unsafeSet(newKey); } +}; + +template <typename T> +struct MovableCellHasher<HeapPtr<T>> +{ + using Key = HeapPtr<T>; + using Lookup = T; + + static bool hasHash(const Lookup& l) { return MovableCellHasher<T>::hasHash(l); } + static bool ensureHash(const Lookup& l) { return MovableCellHasher<T>::ensureHash(l); } + static HashNumber hash(const Lookup& l) { return MovableCellHasher<T>::hash(l); } + static bool match(const Key& k, const Lookup& l) { return MovableCellHasher<T>::match(k, l); } + static void rekey(Key& k, const Key& newKey) { k.unsafeSet(newKey); } +}; + +template <typename T> +struct MovableCellHasher<ReadBarriered<T>> +{ + using Key = ReadBarriered<T>; + using Lookup = T; + + static bool hasHash(const Lookup& l) { return MovableCellHasher<T>::hasHash(l); } + static bool ensureHash(const Lookup& l) { return MovableCellHasher<T>::ensureHash(l); } + static HashNumber hash(const Lookup& l) { return MovableCellHasher<T>::hash(l); } + static bool match(const Key& k, const Lookup& l) { + return MovableCellHasher<T>::match(k.unbarrieredGet(), l); + } + static void rekey(Key& k, const Key& newKey) { k.unsafeSet(newKey); } +}; + +/* Useful for hashtables with a GCPtr as key. */ +template <class T> +struct GCPtrHasher +{ + typedef GCPtr<T> Key; + typedef T Lookup; + + static HashNumber hash(Lookup obj) { return DefaultHasher<T>::hash(obj); } + static bool match(const Key& k, Lookup l) { return k.get() == l; } + static void rekey(Key& k, const Key& newKey) { k.unsafeSet(newKey); } +}; + +/* Specialized hashing policy for GCPtrs. */ +template <class T> +struct DefaultHasher<GCPtr<T>> : GCPtrHasher<T> {}; + +template <class T> +struct PreBarrieredHasher +{ + typedef PreBarriered<T> Key; + typedef T Lookup; + + static HashNumber hash(Lookup obj) { return DefaultHasher<T>::hash(obj); } + static bool match(const Key& k, Lookup l) { return k.get() == l; } + static void rekey(Key& k, const Key& newKey) { k.unsafeSet(newKey); } +}; + +template <class T> +struct DefaultHasher<PreBarriered<T>> : PreBarrieredHasher<T> { }; + +/* Useful for hashtables with a ReadBarriered as key. */ +template <class T> +struct ReadBarrieredHasher +{ + typedef ReadBarriered<T> Key; + typedef T Lookup; + + static HashNumber hash(Lookup obj) { return DefaultHasher<T>::hash(obj); } + static bool match(const Key& k, Lookup l) { return k.unbarrieredGet() == l; } + static void rekey(Key& k, const Key& newKey) { k.set(newKey.unbarrieredGet()); } +}; + +/* Specialized hashing policy for ReadBarriereds. */ +template <class T> +struct DefaultHasher<ReadBarriered<T>> : ReadBarrieredHasher<T> { }; + +class ArrayObject; +class ArrayBufferObject; +class GlobalObject; +class Scope; +class ScriptSourceObject; +class Shape; +class BaseShape; +class UnownedBaseShape; +class WasmInstanceObject; +class WasmTableObject; +namespace jit { +class JitCode; +} // namespace jit + +typedef PreBarriered<JSObject*> PreBarrieredObject; +typedef PreBarriered<JSScript*> PreBarrieredScript; +typedef PreBarriered<jit::JitCode*> PreBarrieredJitCode; +typedef PreBarriered<JSString*> PreBarrieredString; +typedef PreBarriered<JSAtom*> PreBarrieredAtom; + +typedef GCPtr<NativeObject*> GCPtrNativeObject; +typedef GCPtr<ArrayObject*> GCPtrArrayObject; +typedef GCPtr<ArrayBufferObjectMaybeShared*> GCPtrArrayBufferObjectMaybeShared; +typedef GCPtr<ArrayBufferObject*> GCPtrArrayBufferObject; +typedef GCPtr<BaseShape*> GCPtrBaseShape; +typedef GCPtr<JSAtom*> GCPtrAtom; +typedef GCPtr<JSFlatString*> GCPtrFlatString; +typedef GCPtr<JSFunction*> GCPtrFunction; +typedef GCPtr<JSLinearString*> GCPtrLinearString; +typedef GCPtr<JSObject*> GCPtrObject; +typedef GCPtr<JSScript*> GCPtrScript; +typedef GCPtr<JSString*> GCPtrString; +typedef GCPtr<ModuleObject*> GCPtrModuleObject; +typedef GCPtr<ModuleEnvironmentObject*> GCPtrModuleEnvironmentObject; +typedef GCPtr<ModuleNamespaceObject*> GCPtrModuleNamespaceObject; +typedef GCPtr<PlainObject*> GCPtrPlainObject; +typedef GCPtr<PropertyName*> GCPtrPropertyName; +typedef GCPtr<Shape*> GCPtrShape; +typedef GCPtr<UnownedBaseShape*> GCPtrUnownedBaseShape; +typedef GCPtr<jit::JitCode*> GCPtrJitCode; +typedef GCPtr<ObjectGroup*> GCPtrObjectGroup; +typedef GCPtr<Scope*> GCPtrScope; + +typedef PreBarriered<Value> PreBarrieredValue; +typedef GCPtr<Value> GCPtrValue; + +typedef PreBarriered<jsid> PreBarrieredId; +typedef GCPtr<jsid> GCPtrId; + +typedef ImmutableTenuredPtr<PropertyName*> ImmutablePropertyNamePtr; +typedef ImmutableTenuredPtr<JS::Symbol*> ImmutableSymbolPtr; + +typedef ReadBarriered<DebugEnvironmentProxy*> ReadBarrieredDebugEnvironmentProxy; +typedef ReadBarriered<GlobalObject*> ReadBarrieredGlobalObject; +typedef ReadBarriered<JSObject*> ReadBarrieredObject; +typedef ReadBarriered<JSFunction*> ReadBarrieredFunction; +typedef ReadBarriered<JSScript*> ReadBarrieredScript; +typedef ReadBarriered<ScriptSourceObject*> ReadBarrieredScriptSourceObject; +typedef ReadBarriered<Shape*> ReadBarrieredShape; +typedef ReadBarriered<jit::JitCode*> ReadBarrieredJitCode; +typedef ReadBarriered<ObjectGroup*> ReadBarrieredObjectGroup; +typedef ReadBarriered<JS::Symbol*> ReadBarrieredSymbol; +typedef ReadBarriered<WasmInstanceObject*> ReadBarrieredWasmInstanceObject; +typedef ReadBarriered<WasmTableObject*> ReadBarrieredWasmTableObject; + +typedef ReadBarriered<Value> ReadBarrieredValue; + +} /* namespace js */ + +#endif /* gc_Barrier_h */ diff --git a/js/src/gc/FindSCCs.h b/js/src/gc/FindSCCs.h new file mode 100644 index 000000000..037557e3e --- /dev/null +++ b/js/src/gc/FindSCCs.h @@ -0,0 +1,214 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_FindSCCs_h +#define gc_FindSCCs_h + +#include "mozilla/Move.h" + +#include "jsfriendapi.h" +#include "jsutil.h" + +namespace js { +namespace gc { + +template<class Node> +struct GraphNodeBase +{ + Node* gcNextGraphNode; + Node* gcNextGraphComponent; + unsigned gcDiscoveryTime; + unsigned gcLowLink; + + GraphNodeBase() + : gcNextGraphNode(nullptr), + gcNextGraphComponent(nullptr), + gcDiscoveryTime(0), + gcLowLink(0) {} + + ~GraphNodeBase() {} + + Node* nextNodeInGroup() const { + if (gcNextGraphNode && gcNextGraphNode->gcNextGraphComponent == gcNextGraphComponent) + return gcNextGraphNode; + return nullptr; + } + + Node* nextGroup() const { + return gcNextGraphComponent; + } +}; + +/* + * Find the strongly connected components of a graph using Tarjan's algorithm, + * and return them in topological order. + * + * Nodes derive from GraphNodeBase and implement findGraphEdges, which calls + * finder.addEdgeTo to describe the outgoing edges from that node: + * + * struct MyComponentFinder; + * + * struct MyGraphNode : public GraphNodeBase + * { + * void findOutgoingEdges(MyComponentFinder& finder) + * { + * for edge in my_outgoing_edges: + * if is_relevant(edge): + * finder.addEdgeTo(edge.destination) + * } + * } + * + * struct MyComponentFinder : public ComponentFinder<MyGraphNode, MyComponentFinder> + * { + * ... + * }; + * + * MyComponentFinder finder; + * finder.addNode(v); + */ + +template <typename Node, typename Derived> +class ComponentFinder +{ + public: + explicit ComponentFinder(uintptr_t sl) + : clock(1), + stack(nullptr), + firstComponent(nullptr), + cur(nullptr), + stackLimit(sl), + stackFull(false) + {} + + ~ComponentFinder() { + MOZ_ASSERT(!stack); + MOZ_ASSERT(!firstComponent); + } + + /* Forces all nodes to be added to a single component. */ + void useOneComponent() { stackFull = true; } + + void addNode(Node* v) { + if (v->gcDiscoveryTime == Undefined) { + MOZ_ASSERT(v->gcLowLink == Undefined); + processNode(v); + } + } + + Node* getResultsList() { + if (stackFull) { + /* + * All nodes after the stack overflow are in |stack|. Put them all in + * one big component of their own. + */ + Node* firstGoodComponent = firstComponent; + for (Node* v = stack; v; v = stack) { + stack = v->gcNextGraphNode; + v->gcNextGraphComponent = firstGoodComponent; + v->gcNextGraphNode = firstComponent; + firstComponent = v; + } + stackFull = false; + } + + MOZ_ASSERT(!stack); + + Node* result = firstComponent; + firstComponent = nullptr; + + for (Node* v = result; v; v = v->gcNextGraphNode) { + v->gcDiscoveryTime = Undefined; + v->gcLowLink = Undefined; + } + + return result; + } + + static void mergeGroups(Node* first) { + for (Node* v = first; v; v = v->gcNextGraphNode) + v->gcNextGraphComponent = nullptr; + } + + public: + /* Call from implementation of GraphNodeBase::findOutgoingEdges(). */ + void addEdgeTo(Node* w) { + if (w->gcDiscoveryTime == Undefined) { + processNode(w); + cur->gcLowLink = Min(cur->gcLowLink, w->gcLowLink); + } else if (w->gcDiscoveryTime != Finished) { + cur->gcLowLink = Min(cur->gcLowLink, w->gcDiscoveryTime); + } + } + + private: + /* Constant used to indicate an unprocessed vertex. */ + static const unsigned Undefined = 0; + + /* Constant used to indicate an processed vertex that is no longer on the stack. */ + static const unsigned Finished = (unsigned)-1; + + void processNode(Node* v) { + v->gcDiscoveryTime = clock; + v->gcLowLink = clock; + ++clock; + + v->gcNextGraphNode = stack; + stack = v; + + int stackDummy; + if (stackFull || !JS_CHECK_STACK_SIZE(stackLimit, &stackDummy)) { + stackFull = true; + return; + } + + Node* old = cur; + cur = v; + cur->findOutgoingEdges(*static_cast<Derived*>(this)); + cur = old; + + if (stackFull) + return; + + if (v->gcLowLink == v->gcDiscoveryTime) { + Node* nextComponent = firstComponent; + Node* w; + do { + MOZ_ASSERT(stack); + w = stack; + stack = w->gcNextGraphNode; + + /* + * Record that the element is no longer on the stack by setting the + * discovery time to a special value that's not Undefined. + */ + w->gcDiscoveryTime = Finished; + + /* Figure out which group we're in. */ + w->gcNextGraphComponent = nextComponent; + + /* + * Prepend the component to the beginning of the output list to + * reverse the list and achieve the desired order. + */ + w->gcNextGraphNode = firstComponent; + firstComponent = w; + } while (w != v); + } + } + + private: + unsigned clock; + Node* stack; + Node* firstComponent; + Node* cur; + uintptr_t stackLimit; + bool stackFull; +}; + +} /* namespace gc */ +} /* namespace js */ + +#endif /* gc_FindSCCs_h */ diff --git a/js/src/gc/GCInternals.h b/js/src/gc/GCInternals.h new file mode 100644 index 000000000..722539e1c --- /dev/null +++ b/js/src/gc/GCInternals.h @@ -0,0 +1,175 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_GCInternals_h +#define gc_GCInternals_h + +#include "mozilla/ArrayUtils.h" +#include "mozilla/Maybe.h" +#include "mozilla/PodOperations.h" + +#include "jscntxt.h" + +#include "gc/Zone.h" +#include "vm/HelperThreads.h" +#include "vm/Runtime.h" + +namespace js { +namespace gc { + +void FinishGC(JSContext* cx); + +/* + * This class should be used by any code that needs to exclusive access to the + * heap in order to trace through it... + */ +class MOZ_RAII AutoTraceSession +{ + public: + explicit AutoTraceSession(JSRuntime* rt, JS::HeapState state = JS::HeapState::Tracing); + ~AutoTraceSession(); + + // Threads with an exclusive context can hit refillFreeList while holding + // the exclusive access lock. To avoid deadlocking when we try to acquire + // this lock during GC and the other thread is waiting, make sure we hold + // the exclusive access lock during GC sessions. + AutoLockForExclusiveAccess lock; + + protected: + JSRuntime* runtime; + + private: + AutoTraceSession(const AutoTraceSession&) = delete; + void operator=(const AutoTraceSession&) = delete; + + JS::HeapState prevState; + AutoSPSEntry pseudoFrame; +}; + +class MOZ_RAII AutoPrepareForTracing +{ + mozilla::Maybe<AutoTraceSession> session_; + + public: + AutoPrepareForTracing(JSContext* cx, ZoneSelector selector); + AutoTraceSession& session() { return session_.ref(); } +}; + +AbortReason +IsIncrementalGCUnsafe(JSRuntime* rt); + +#ifdef JS_GC_ZEAL + +class MOZ_RAII AutoStopVerifyingBarriers +{ + GCRuntime* gc; + bool restartPreVerifier; + + public: + AutoStopVerifyingBarriers(JSRuntime* rt, bool isShutdown) + : gc(&rt->gc) + { + if (gc->isVerifyPreBarriersEnabled()) { + gc->endVerifyPreBarriers(); + restartPreVerifier = !isShutdown; + } else { + restartPreVerifier = false; + } + } + + ~AutoStopVerifyingBarriers() { + // Nasty special case: verification runs a minor GC, which *may* nest + // inside of an outer minor GC. This is not allowed by the + // gc::Statistics phase tree. So we pause the "real" GC, if in fact one + // is in progress. + gcstats::Phase outer = gc->stats.currentPhase(); + if (outer != gcstats::PHASE_NONE) + gc->stats.endPhase(outer); + MOZ_ASSERT((gc->stats.currentPhase() == gcstats::PHASE_NONE) || + (gc->stats.currentPhase() == gcstats::PHASE_GC_BEGIN) || + (gc->stats.currentPhase() == gcstats::PHASE_GC_END)); + + if (restartPreVerifier) + gc->startVerifyPreBarriers(); + + if (outer != gcstats::PHASE_NONE) + gc->stats.beginPhase(outer); + } +}; +#else +struct MOZ_RAII AutoStopVerifyingBarriers +{ + AutoStopVerifyingBarriers(JSRuntime*, bool) {} +}; +#endif /* JS_GC_ZEAL */ + +#ifdef JSGC_HASH_TABLE_CHECKS +void CheckHashTablesAfterMovingGC(JSRuntime* rt); +void CheckHeapAfterGC(JSRuntime* rt); +#endif + +struct MovingTracer : JS::CallbackTracer +{ + explicit MovingTracer(JSRuntime* rt) : CallbackTracer(rt, TraceWeakMapKeysValues) {} + + void onObjectEdge(JSObject** objp) override; + void onShapeEdge(Shape** shapep) override; + void onStringEdge(JSString** stringp) override; + void onScriptEdge(JSScript** scriptp) override; + void onLazyScriptEdge(LazyScript** lazyp) override; + void onBaseShapeEdge(BaseShape** basep) override; + void onScopeEdge(Scope** basep) override; + void onChild(const JS::GCCellPtr& thing) override { + MOZ_ASSERT(!RelocationOverlay::isCellForwarded(thing.asCell())); + } + +#ifdef DEBUG + TracerKind getTracerKind() const override { return TracerKind::Moving; } +#endif +}; + +// Structure for counting how many times objects in a particular group have +// been tenured during a minor collection. +struct TenureCount +{ + ObjectGroup* group; + int count; +}; + +// Keep rough track of how many times we tenure objects in particular groups +// during minor collections, using a fixed size hash for efficiency at the cost +// of potential collisions. +struct TenureCountCache +{ + static const size_t EntryShift = 4; + static const size_t EntryCount = 1 << EntryShift; + + TenureCount entries[EntryCount]; + + TenureCountCache() { mozilla::PodZero(this); } + + HashNumber hash(ObjectGroup* group) { +#if JS_BITS_PER_WORD == 32 + static const size_t ZeroBits = 3; +#else + static const size_t ZeroBits = 4; +#endif + + uintptr_t word = uintptr_t(group); + MOZ_ASSERT((word & ((1 << ZeroBits) - 1)) == 0); + word >>= ZeroBits; + return HashNumber((word >> EntryShift) ^ word); + } + + TenureCount& findEntry(ObjectGroup* group) { + return entries[hash(group) % EntryCount]; + } +}; + +} /* namespace gc */ +} /* namespace js */ + +#endif /* gc_GCInternals_h */ diff --git a/js/src/gc/GCRuntime.h b/js/src/gc/GCRuntime.h new file mode 100644 index 000000000..8c9322849 --- /dev/null +++ b/js/src/gc/GCRuntime.h @@ -0,0 +1,1467 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_GCRuntime_h +#define gc_GCRuntime_h + +#include "mozilla/Atomics.h" +#include "mozilla/EnumSet.h" + +#include "jsfriendapi.h" +#include "jsgc.h" + +#include "gc/Heap.h" +#include "gc/Nursery.h" +#include "gc/Statistics.h" +#include "gc/StoreBuffer.h" +#include "gc/Tracer.h" +#include "js/GCAnnotations.h" + +namespace js { + +class AutoLockGC; +class AutoLockHelperThreadState; +class VerifyPreTracer; + +namespace gc { + +typedef Vector<JS::Zone*, 4, SystemAllocPolicy> ZoneVector; +using BlackGrayEdgeVector = Vector<TenuredCell*, 0, SystemAllocPolicy>; + +class AutoMaybeStartBackgroundAllocation; +class MarkingValidator; +class AutoTraceSession; +struct MovingTracer; + +class ChunkPool +{ + Chunk* head_; + size_t count_; + + public: + ChunkPool() : head_(nullptr), count_(0) {} + + size_t count() const { return count_; } + + Chunk* head() { MOZ_ASSERT(head_); return head_; } + Chunk* pop(); + void push(Chunk* chunk); + Chunk* remove(Chunk* chunk); + +#ifdef DEBUG + bool contains(Chunk* chunk) const; + bool verify() const; +#endif + + // Pool mutation does not invalidate an Iter unless the mutation + // is of the Chunk currently being visited by the Iter. + class Iter { + public: + explicit Iter(ChunkPool& pool) : current_(pool.head_) {} + bool done() const { return !current_; } + void next(); + Chunk* get() const { return current_; } + operator Chunk*() const { return get(); } + Chunk* operator->() const { return get(); } + private: + Chunk* current_; + }; +}; + +// Performs extra allocation off the main thread so that when memory is +// required on the main thread it will already be available and waiting. +class BackgroundAllocTask : public GCParallelTask +{ + // Guarded by the GC lock. + JSRuntime* runtime; + ChunkPool& chunkPool_; + + const bool enabled_; + + public: + BackgroundAllocTask(JSRuntime* rt, ChunkPool& pool); + bool enabled() const { return enabled_; } + + protected: + void run() override; +}; + +// Search the provided Chunks for free arenas and decommit them. +class BackgroundDecommitTask : public GCParallelTask +{ + public: + using ChunkVector = mozilla::Vector<Chunk*>; + + explicit BackgroundDecommitTask(JSRuntime *rt) : runtime(rt) {} + void setChunksToScan(ChunkVector &chunks); + + protected: + void run() override; + + private: + JSRuntime* runtime; + ChunkVector toDecommit; +}; + +/* + * Encapsulates all of the GC tunables. These are effectively constant and + * should only be modified by setParameter. + */ +class GCSchedulingTunables +{ + /* + * Soft limit on the number of bytes we are allowed to allocate in the GC + * heap. Attempts to allocate gcthings over this limit will return null and + * subsequently invoke the standard OOM machinery, independent of available + * physical memory. + */ + size_t gcMaxBytes_; + + /* + * The base value used to compute zone->trigger.gcBytes(). When + * usage.gcBytes() surpasses threshold.gcBytes() for a zone, the zone may + * be scheduled for a GC, depending on the exact circumstances. + */ + size_t gcZoneAllocThresholdBase_; + + /* Fraction of threshold.gcBytes() which triggers an incremental GC. */ + double zoneAllocThresholdFactor_; + + /* + * Number of bytes to allocate between incremental slices in GCs triggered + * by the zone allocation threshold. + */ + size_t zoneAllocDelayBytes_; + + /* + * Totally disables |highFrequencyGC|, the HeapGrowthFactor, and other + * tunables that make GC non-deterministic. + */ + bool dynamicHeapGrowthEnabled_; + + /* + * We enter high-frequency mode if we GC a twice within this many + * microseconds. This value is stored directly in microseconds. + */ + uint64_t highFrequencyThresholdUsec_; + + /* + * When in the |highFrequencyGC| mode, these parameterize the per-zone + * "HeapGrowthFactor" computation. + */ + uint64_t highFrequencyLowLimitBytes_; + uint64_t highFrequencyHighLimitBytes_; + double highFrequencyHeapGrowthMax_; + double highFrequencyHeapGrowthMin_; + + /* + * When not in |highFrequencyGC| mode, this is the global (stored per-zone) + * "HeapGrowthFactor". + */ + double lowFrequencyHeapGrowth_; + + /* + * Doubles the length of IGC slices when in the |highFrequencyGC| mode. + */ + bool dynamicMarkSliceEnabled_; + + /* + * Controls whether painting can trigger IGC slices. + */ + bool refreshFrameSlicesEnabled_; + + /* + * Controls the number of empty chunks reserved for future allocation. + */ + uint32_t minEmptyChunkCount_; + uint32_t maxEmptyChunkCount_; + + public: + GCSchedulingTunables() + : gcMaxBytes_(0), + gcZoneAllocThresholdBase_(30 * 1024 * 1024), + zoneAllocThresholdFactor_(0.9), + zoneAllocDelayBytes_(1024 * 1024), + dynamicHeapGrowthEnabled_(false), + highFrequencyThresholdUsec_(1000 * 1000), + highFrequencyLowLimitBytes_(100 * 1024 * 1024), + highFrequencyHighLimitBytes_(500 * 1024 * 1024), + highFrequencyHeapGrowthMax_(3.0), + highFrequencyHeapGrowthMin_(1.5), + lowFrequencyHeapGrowth_(1.5), + dynamicMarkSliceEnabled_(false), + refreshFrameSlicesEnabled_(true), + minEmptyChunkCount_(1), + maxEmptyChunkCount_(30) + {} + + size_t gcMaxBytes() const { return gcMaxBytes_; } + size_t gcZoneAllocThresholdBase() const { return gcZoneAllocThresholdBase_; } + double zoneAllocThresholdFactor() const { return zoneAllocThresholdFactor_; } + size_t zoneAllocDelayBytes() const { return zoneAllocDelayBytes_; } + bool isDynamicHeapGrowthEnabled() const { return dynamicHeapGrowthEnabled_; } + uint64_t highFrequencyThresholdUsec() const { return highFrequencyThresholdUsec_; } + uint64_t highFrequencyLowLimitBytes() const { return highFrequencyLowLimitBytes_; } + uint64_t highFrequencyHighLimitBytes() const { return highFrequencyHighLimitBytes_; } + double highFrequencyHeapGrowthMax() const { return highFrequencyHeapGrowthMax_; } + double highFrequencyHeapGrowthMin() const { return highFrequencyHeapGrowthMin_; } + double lowFrequencyHeapGrowth() const { return lowFrequencyHeapGrowth_; } + bool isDynamicMarkSliceEnabled() const { return dynamicMarkSliceEnabled_; } + bool areRefreshFrameSlicesEnabled() const { return refreshFrameSlicesEnabled_; } + unsigned minEmptyChunkCount(const AutoLockGC&) const { return minEmptyChunkCount_; } + unsigned maxEmptyChunkCount() const { return maxEmptyChunkCount_; } + + MOZ_MUST_USE bool setParameter(JSGCParamKey key, uint32_t value, const AutoLockGC& lock); +}; + +/* + * GC Scheduling Overview + * ====================== + * + * Scheduling GC's in SpiderMonkey/Firefox is tremendously complicated because + * of the large number of subtle, cross-cutting, and widely dispersed factors + * that must be taken into account. A summary of some of the more important + * factors follows. + * + * Cost factors: + * + * * GC too soon and we'll revisit an object graph almost identical to the + * one we just visited; since we are unlikely to find new garbage, the + * traversal will be largely overhead. We rely heavily on external factors + * to signal us that we are likely to find lots of garbage: e.g. "a tab + * just got closed". + * + * * GC too late and we'll run out of memory to allocate (e.g. Out-Of-Memory, + * hereafter simply abbreviated to OOM). If this happens inside + * SpiderMonkey we may be able to recover, but most embedder allocations + * will simply crash on OOM, even if the GC has plenty of free memory it + * could surrender. + * + * * Memory fragmentation: if we fill the process with GC allocations, a + * request for a large block of contiguous memory may fail because no + * contiguous block is free, despite having enough memory available to + * service the request. + * + * * Management overhead: if our GC heap becomes large, we create extra + * overhead when managing the GC's structures, even if the allocations are + * mostly unused. + * + * Heap Management Factors: + * + * * GC memory: The GC has its own allocator that it uses to make fixed size + * allocations for GC managed things. In cases where the GC thing requires + * larger or variable sized memory to implement itself, it is responsible + * for using the system heap. + * + * * C Heap Memory: Rather than allowing for large or variable allocations, + * the SpiderMonkey GC allows GC things to hold pointers to C heap memory. + * It is the responsibility of the thing to free this memory with a custom + * finalizer (with the sole exception of NativeObject, which knows about + * slots and elements for performance reasons). C heap memory has different + * performance and overhead tradeoffs than GC internal memory, which need + * to be considered with scheduling a GC. + * + * Application Factors: + * + * * Most applications allocate heavily at startup, then enter a processing + * stage where memory utilization remains roughly fixed with a slower + * allocation rate. This is not always the case, however, so while we may + * optimize for this pattern, we must be able to handle arbitrary + * allocation patterns. + * + * Other factors: + * + * * Other memory: This is memory allocated outside the purview of the GC. + * Data mapped by the system for code libraries, data allocated by those + * libraries, data in the JSRuntime that is used to manage the engine, + * memory used by the embedding that is not attached to a GC thing, memory + * used by unrelated processes running on the hardware that use space we + * could otherwise use for allocation, etc. While we don't have to manage + * it, we do have to take it into account when scheduling since it affects + * when we will OOM. + * + * * Physical Reality: All real machines have limits on the number of bits + * that they are physically able to store. While modern operating systems + * can generally make additional space available with swapping, at some + * point there are simply no more bits to allocate. There is also the + * factor of address space limitations, particularly on 32bit machines. + * + * * Platform Factors: Each OS makes use of wildly different memory + * management techniques. These differences result in different performance + * tradeoffs, different fragmentation patterns, and different hard limits + * on the amount of physical and/or virtual memory that we can use before + * OOMing. + * + * + * Reasons for scheduling GC + * ------------------------- + * + * While code generally takes the above factors into account in only an ad-hoc + * fashion, the API forces the user to pick a "reason" for the GC. We have a + * bunch of JS::gcreason reasons in GCAPI.h. These fall into a few categories + * that generally coincide with one or more of the above factors. + * + * Embedding reasons: + * + * 1) Do a GC now because the embedding knows something useful about the + * zone's memory retention state. These are gcreasons like LOAD_END, + * PAGE_HIDE, SET_NEW_DOCUMENT, DOM_UTILS. Mostly, Gecko uses these to + * indicate that a significant fraction of the scheduled zone's memory is + * probably reclaimable. + * + * 2) Do some known amount of GC work now because the embedding knows now is + * a good time to do a long, unblockable operation of a known duration. + * These are INTER_SLICE_GC and REFRESH_FRAME. + * + * Correctness reasons: + * + * 3) Do a GC now because correctness depends on some GC property. For + * example, CC_WAITING is where the embedding requires the mark bits + * to be set correct. Also, EVICT_NURSERY where we need to work on the tenured + * heap. + * + * 4) Do a GC because we are shutting down: e.g. SHUTDOWN_CC or DESTROY_*. + * + * 5) Do a GC because a compartment was accessed between GC slices when we + * would have otherwise discarded it. We have to do a second GC to clean + * it up: e.g. COMPARTMENT_REVIVED. + * + * Emergency Reasons: + * + * 6) Do an all-zones, non-incremental GC now because the embedding knows it + * cannot wait: e.g. MEM_PRESSURE. + * + * 7) OOM when fetching a new Chunk results in a LAST_DITCH GC. + * + * Heap Size Limitation Reasons: + * + * 8) Do an incremental, zonal GC with reason MAYBEGC when we discover that + * the gc's allocated size is approaching the current trigger. This is + * called MAYBEGC because we make this check in the MaybeGC function. + * MaybeGC gets called at the top of the main event loop. Normally, it is + * expected that this callback will keep the heap size limited. It is + * relatively inexpensive, because it is invoked with no JS running and + * thus few stack roots to scan. For this reason, the GC's "trigger" bytes + * is less than the GC's "max" bytes as used by the trigger below. + * + * 9) Do an incremental, zonal GC with reason MAYBEGC when we go to allocate + * a new GC thing and find that the GC heap size has grown beyond the + * configured maximum (JSGC_MAX_BYTES). We trigger this GC by returning + * nullptr and then calling maybeGC at the top level of the allocator. + * This is then guaranteed to fail the "size greater than trigger" check + * above, since trigger is always less than max. After performing the GC, + * the allocator unconditionally returns nullptr to force an OOM exception + * is raised by the script. + * + * Note that this differs from a LAST_DITCH GC where we actually run out + * of memory (i.e., a call to a system allocator fails) when trying to + * allocate. Unlike above, LAST_DITCH GC only happens when we are really + * out of memory, not just when we cross an arbitrary trigger; despite + * this, it may still return an allocation at the end and allow the script + * to continue, if the LAST_DITCH GC was able to free up enough memory. + * + * 10) Do a GC under reason ALLOC_TRIGGER when we are over the GC heap trigger + * limit, but in the allocator rather than in a random call to maybeGC. + * This occurs if we allocate too much before returning to the event loop + * and calling maybeGC; this is extremely common in benchmarks and + * long-running Worker computations. Note that this uses a wildly + * different mechanism from the above in that it sets the interrupt flag + * and does the GC at the next loop head, before the next alloc, or + * maybeGC. The reason for this is that this check is made after the + * allocation and we cannot GC with an uninitialized thing in the heap. + * + * 11) Do an incremental, zonal GC with reason TOO_MUCH_MALLOC when we have + * malloced more than JSGC_MAX_MALLOC_BYTES in a zone since the last GC. + * + * + * Size Limitation Triggers Explanation + * ------------------------------------ + * + * The GC internally is entirely unaware of the context of the execution of + * the mutator. It sees only: + * + * A) Allocated size: this is the amount of memory currently requested by the + * mutator. This quantity is monotonically increasing: i.e. the allocation + * rate is always >= 0. It is also easy for the system to track. + * + * B) Retained size: this is the amount of memory that the mutator can + * currently reach. Said another way, it is the size of the heap + * immediately after a GC (modulo background sweeping). This size is very + * costly to know exactly and also extremely hard to estimate with any + * fidelity. + * + * For reference, a common allocated vs. retained graph might look like: + * + * | ** ** + * | ** ** * ** + * | ** * ** * ** + * | * ** * ** * ** + * | ** ** * ** * ** + * s| * * ** ** + + ** + * i| * * * + + + + + + * z| * * * + + + + + + * e| * **+ + * | * + + * | * + + * | * + + * | * + + * | * + + * |*+ + * +-------------------------------------------------- + * time + * *** = allocated + * +++ = retained + * + * Note that this is a bit of a simplification + * because in reality we track malloc and GC heap + * sizes separately and have a different level of + * granularity and accuracy on each heap. + * + * This presents some obvious implications for Mark-and-Sweep collectors. + * Namely: + * -> t[marking] ~= size[retained] + * -> t[sweeping] ~= size[allocated] - size[retained] + * + * In a non-incremental collector, maintaining low latency and high + * responsiveness requires that total GC times be as low as possible. Thus, + * in order to stay responsive when we did not have a fully incremental + * collector, our GC triggers were focused on minimizing collection time. + * Furthermore, since size[retained] is not under control of the GC, all the + * GC could do to control collection times was reduce sweep times by + * minimizing size[allocated], per the equation above. + * + * The result of the above is GC triggers that focus on size[allocated] to + * the exclusion of other important factors and default heuristics that are + * not optimal for a fully incremental collector. On the other hand, this is + * not all bad: minimizing size[allocated] also minimizes the chance of OOM + * and sweeping remains one of the hardest areas to further incrementalize. + * + * EAGER_ALLOC_TRIGGER + * ------------------- + * Occurs when we return to the event loop and find our heap is getting + * largish, but before t[marking] OR t[sweeping] is too large for a + * responsive non-incremental GC. This is intended to be the common case + * in normal web applications: e.g. we just finished an event handler and + * the few objects we allocated when computing the new whatzitz have + * pushed us slightly over the limit. After this GC we rescale the new + * EAGER_ALLOC_TRIGGER trigger to 150% of size[retained] so that our + * non-incremental GC times will always be proportional to this size + * rather than being dominated by sweeping. + * + * As a concession to mutators that allocate heavily during their startup + * phase, we have a highFrequencyGCMode that ups the growth rate to 300% + * of the current size[retained] so that we'll do fewer longer GCs at the + * end of the mutator startup rather than more, smaller GCs. + * + * Assumptions: + * -> Responsiveness is proportional to t[marking] + t[sweeping]. + * -> size[retained] is proportional only to GC allocations. + * + * ALLOC_TRIGGER (non-incremental) + * ------------------------------- + * If we do not return to the event loop before getting all the way to our + * gc trigger bytes then MAYBEGC will never fire. To avoid OOMing, we + * succeed the current allocation and set the script interrupt so that we + * will (hopefully) do a GC before we overflow our max and have to raise + * an OOM exception for the script. + * + * Assumptions: + * -> Common web scripts will return to the event loop before using + * 10% of the current gcTriggerBytes worth of GC memory. + * + * ALLOC_TRIGGER (incremental) + * --------------------------- + * In practice the above trigger is rough: if a website is just on the + * cusp, sometimes it will trigger a non-incremental GC moments before + * returning to the event loop, where it could have done an incremental + * GC. Thus, we recently added an incremental version of the above with a + * substantially lower threshold, so that we have a soft limit here. If + * IGC can collect faster than the allocator generates garbage, even if + * the allocator does not return to the event loop frequently, we should + * not have to fall back to a non-incremental GC. + * + * INCREMENTAL_TOO_SLOW + * -------------------- + * Do a full, non-incremental GC if we overflow ALLOC_TRIGGER during an + * incremental GC. When in the middle of an incremental GC, we suppress + * our other triggers, so we need a way to backstop the IGC if the + * mutator allocates faster than the IGC can clean things up. + * + * TOO_MUCH_MALLOC + * --------------- + * Performs a GC before size[allocated] - size[retained] gets too large + * for non-incremental sweeping to be fast in the case that we have + * significantly more malloc allocation than GC allocation. This is meant + * to complement MAYBEGC triggers. We track this by counting malloced + * bytes; the counter gets reset at every GC since we do not always have a + * size at the time we call free. Because of this, the malloc heuristic + * is, unfortunatly, not usefully able to augment our other GC heap + * triggers and is limited to this singular heuristic. + * + * Assumptions: + * -> EITHER size[allocated_by_malloc] ~= size[allocated_by_GC] + * OR time[sweeping] ~= size[allocated_by_malloc] + * -> size[retained] @ t0 ~= size[retained] @ t1 + * i.e. That the mutator is in steady-state operation. + * + * LAST_DITCH_GC + * ------------- + * Does a GC because we are out of memory. + * + * Assumptions: + * -> size[retained] < size[available_memory] + */ +class GCSchedulingState +{ + /* + * Influences how we schedule and run GC's in several subtle ways. The most + * important factor is in how it controls the "HeapGrowthFactor". The + * growth factor is a measure of how large (as a percentage of the last GC) + * the heap is allowed to grow before we try to schedule another GC. + */ + bool inHighFrequencyGCMode_; + + public: + GCSchedulingState() + : inHighFrequencyGCMode_(false) + {} + + bool inHighFrequencyGCMode() const { return inHighFrequencyGCMode_; } + + void updateHighFrequencyMode(uint64_t lastGCTime, uint64_t currentTime, + const GCSchedulingTunables& tunables) { + inHighFrequencyGCMode_ = + tunables.isDynamicHeapGrowthEnabled() && lastGCTime && + lastGCTime + tunables.highFrequencyThresholdUsec() > currentTime; + } +}; + +template<typename F> +struct Callback { + F op; + void* data; + + Callback() + : op(nullptr), data(nullptr) + {} + Callback(F op, void* data) + : op(op), data(data) + {} +}; + +template<typename F> +using CallbackVector = Vector<Callback<F>, 4, SystemAllocPolicy>; + +template <typename T, typename Iter0, typename Iter1> +class ChainedIter +{ + Iter0 iter0_; + Iter1 iter1_; + + public: + ChainedIter(const Iter0& iter0, const Iter1& iter1) + : iter0_(iter0), iter1_(iter1) + {} + + bool done() const { return iter0_.done() && iter1_.done(); } + void next() { + MOZ_ASSERT(!done()); + if (!iter0_.done()) { + iter0_.next(); + } else { + MOZ_ASSERT(!iter1_.done()); + iter1_.next(); + } + } + T get() const { + MOZ_ASSERT(!done()); + if (!iter0_.done()) + return iter0_.get(); + MOZ_ASSERT(!iter1_.done()); + return iter1_.get(); + } + + operator T() const { return get(); } + T operator->() const { return get(); } +}; + +typedef HashMap<Value*, const char*, DefaultHasher<Value*>, SystemAllocPolicy> RootedValueMap; + +using AllocKinds = mozilla::EnumSet<AllocKind>; + +class GCRuntime +{ + public: + explicit GCRuntime(JSRuntime* rt); + MOZ_MUST_USE bool init(uint32_t maxbytes, uint32_t maxNurseryBytes); + void finishRoots(); + void finish(); + + inline bool hasZealMode(ZealMode mode); + inline void clearZealMode(ZealMode mode); + inline bool upcomingZealousGC(); + inline bool needZealousGC(); + + MOZ_MUST_USE bool addRoot(Value* vp, const char* name); + void removeRoot(Value* vp); + void setMarkStackLimit(size_t limit, AutoLockGC& lock); + + MOZ_MUST_USE bool setParameter(JSGCParamKey key, uint32_t value, AutoLockGC& lock); + uint32_t getParameter(JSGCParamKey key, const AutoLockGC& lock); + + MOZ_MUST_USE bool triggerGC(JS::gcreason::Reason reason); + void maybeAllocTriggerZoneGC(Zone* zone, const AutoLockGC& lock); + // The return value indicates if we were able to do the GC. + bool triggerZoneGC(Zone* zone, JS::gcreason::Reason reason); + void maybeGC(Zone* zone); + void minorGC(JS::gcreason::Reason reason, + gcstats::Phase phase = gcstats::PHASE_MINOR_GC) JS_HAZ_GC_CALL; + void evictNursery(JS::gcreason::Reason reason = JS::gcreason::EVICT_NURSERY) { + minorGC(reason, gcstats::PHASE_EVICT_NURSERY); + } + // The return value indicates whether a major GC was performed. + bool gcIfRequested(); + void gc(JSGCInvocationKind gckind, JS::gcreason::Reason reason); + void startGC(JSGCInvocationKind gckind, JS::gcreason::Reason reason, int64_t millis = 0); + void gcSlice(JS::gcreason::Reason reason, int64_t millis = 0); + void finishGC(JS::gcreason::Reason reason); + void abortGC(); + void startDebugGC(JSGCInvocationKind gckind, SliceBudget& budget); + void debugGCSlice(SliceBudget& budget); + + void triggerFullGCForAtoms() { + MOZ_ASSERT(fullGCForAtomsRequested_); + fullGCForAtomsRequested_ = false; + MOZ_RELEASE_ASSERT(triggerGC(JS::gcreason::ALLOC_TRIGGER)); + } + + void runDebugGC(); + inline void poke(); + + enum TraceOrMarkRuntime { + TraceRuntime, + MarkRuntime + }; + void traceRuntime(JSTracer* trc, AutoLockForExclusiveAccess& lock); + void traceRuntimeForMinorGC(JSTracer* trc, AutoLockForExclusiveAccess& lock); + + void notifyDidPaint(); + void shrinkBuffers(); + void onOutOfMallocMemory(); + void onOutOfMallocMemory(const AutoLockGC& lock); + +#ifdef JS_GC_ZEAL + const void* addressOfZealModeBits() { return &zealModeBits; } + void getZealBits(uint32_t* zealBits, uint32_t* frequency, uint32_t* nextScheduled); + void setZeal(uint8_t zeal, uint32_t frequency); + bool parseAndSetZeal(const char* str); + void setNextScheduled(uint32_t count); + void verifyPreBarriers(); + void maybeVerifyPreBarriers(bool always); + bool selectForMarking(JSObject* object); + void clearSelectedForMarking(); + void setDeterministic(bool enable); +#endif + + size_t maxMallocBytesAllocated() { return maxMallocBytes; } + + uint64_t nextCellUniqueId() { + MOZ_ASSERT(nextCellUniqueId_ > 0); + uint64_t uid = ++nextCellUniqueId_; + return uid; + } + +#ifdef DEBUG + bool shutdownCollectedEverything() const { + return arenasEmptyAtShutdown; + } +#endif + + public: + // Internal public interface + State state() const { return incrementalState; } + bool isHeapCompacting() const { return state() == State::Compact; } + bool isForegroundSweeping() const { return state() == State::Sweep; } + bool isBackgroundSweeping() { return helperState.isBackgroundSweeping(); } + void waitBackgroundSweepEnd() { helperState.waitBackgroundSweepEnd(); } + void waitBackgroundSweepOrAllocEnd() { + helperState.waitBackgroundSweepEnd(); + allocTask.cancel(GCParallelTask::CancelAndWait); + } + + void requestMinorGC(JS::gcreason::Reason reason); + +#ifdef DEBUG + bool onBackgroundThread() { return helperState.onBackgroundThread(); } +#endif // DEBUG + + void lockGC() { + lock.lock(); + } + + void unlockGC() { + lock.unlock(); + } + +#ifdef DEBUG + bool isAllocAllowed() { return noGCOrAllocationCheck == 0; } + void disallowAlloc() { ++noGCOrAllocationCheck; } + void allowAlloc() { + MOZ_ASSERT(!isAllocAllowed()); + --noGCOrAllocationCheck; + } + + bool isNurseryAllocAllowed() { return noNurseryAllocationCheck == 0; } + void disallowNurseryAlloc() { ++noNurseryAllocationCheck; } + void allowNurseryAlloc() { + MOZ_ASSERT(!isNurseryAllocAllowed()); + --noNurseryAllocationCheck; + } + + bool isStrictProxyCheckingEnabled() { return disableStrictProxyCheckingCount == 0; } + void disableStrictProxyChecking() { ++disableStrictProxyCheckingCount; } + void enableStrictProxyChecking() { + MOZ_ASSERT(disableStrictProxyCheckingCount > 0); + --disableStrictProxyCheckingCount; + } +#endif // DEBUG + + bool isInsideUnsafeRegion() { return inUnsafeRegion != 0; } + void enterUnsafeRegion() { ++inUnsafeRegion; } + void leaveUnsafeRegion() { + MOZ_ASSERT(inUnsafeRegion > 0); + --inUnsafeRegion; + } + + void verifyIsSafeToGC() { + MOZ_DIAGNOSTIC_ASSERT(!isInsideUnsafeRegion(), + "[AutoAssertNoGC] possible GC in GC-unsafe region"); + } + + void setAlwaysPreserveCode() { alwaysPreserveCode = true; } + + bool isIncrementalGCAllowed() const { return incrementalAllowed; } + void disallowIncrementalGC() { incrementalAllowed = false; } + + bool isIncrementalGCEnabled() const { return mode == JSGC_MODE_INCREMENTAL && incrementalAllowed; } + bool isIncrementalGCInProgress() const { return state() != State::NotActive; } + + bool isGenerationalGCEnabled() const { return generationalDisabled == 0; } + void disableGenerationalGC(); + void enableGenerationalGC(); + + void disableCompactingGC(); + void enableCompactingGC(); + bool isCompactingGCEnabled() const; + + void setGrayRootsTracer(JSTraceDataOp traceOp, void* data); + MOZ_MUST_USE bool addBlackRootsTracer(JSTraceDataOp traceOp, void* data); + void removeBlackRootsTracer(JSTraceDataOp traceOp, void* data); + + void setMaxMallocBytes(size_t value); + int32_t getMallocBytes() const { return mallocBytesUntilGC; } + void resetMallocBytes(); + bool isTooMuchMalloc() const { return mallocBytesUntilGC <= 0; } + void updateMallocCounter(JS::Zone* zone, size_t nbytes); + void onTooMuchMalloc(); + + void setGCCallback(JSGCCallback callback, void* data); + void callGCCallback(JSGCStatus status) const; + void setObjectsTenuredCallback(JSObjectsTenuredCallback callback, + void* data); + void callObjectsTenuredCallback(); + MOZ_MUST_USE bool addFinalizeCallback(JSFinalizeCallback callback, void* data); + void removeFinalizeCallback(JSFinalizeCallback func); + MOZ_MUST_USE bool addWeakPointerZoneGroupCallback(JSWeakPointerZoneGroupCallback callback, + void* data); + void removeWeakPointerZoneGroupCallback(JSWeakPointerZoneGroupCallback callback); + MOZ_MUST_USE bool addWeakPointerCompartmentCallback(JSWeakPointerCompartmentCallback callback, + void* data); + void removeWeakPointerCompartmentCallback(JSWeakPointerCompartmentCallback callback); + JS::GCSliceCallback setSliceCallback(JS::GCSliceCallback callback); + JS::GCNurseryCollectionCallback setNurseryCollectionCallback( + JS::GCNurseryCollectionCallback callback); + JS::DoCycleCollectionCallback setDoCycleCollectionCallback(JS::DoCycleCollectionCallback callback); + void callDoCycleCollectionCallback(JSContext* cx); + + void setFullCompartmentChecks(bool enable); + + bool isManipulatingDeadZones() { return manipulatingDeadZones; } + void setManipulatingDeadZones(bool value) { manipulatingDeadZones = value; } + unsigned objectsMarkedInDeadZonesCount() { return objectsMarkedInDeadZones; } + void incObjectsMarkedInDeadZone() { + MOZ_ASSERT(manipulatingDeadZones); + ++objectsMarkedInDeadZones; + } + + JS::Zone* getCurrentZoneGroup() { return currentZoneGroup; } + void setFoundBlackGrayEdges(TenuredCell& target) { + AutoEnterOOMUnsafeRegion oomUnsafe; + if (!foundBlackGrayEdges.append(&target)) + oomUnsafe.crash("OOM|small: failed to insert into foundBlackGrayEdges"); + } + + uint64_t gcNumber() const { return number; } + + uint64_t minorGCCount() const { return minorGCNumber; } + void incMinorGcNumber() { ++minorGCNumber; ++number; } + + uint64_t majorGCCount() const { return majorGCNumber; } + void incMajorGcNumber() { ++majorGCNumber; ++number; } + + int64_t defaultSliceBudget() const { return defaultTimeBudget_; } + + bool isIncrementalGc() const { return isIncremental; } + bool isFullGc() const { return isFull; } + bool isCompactingGc() const { return isCompacting; } + + bool minorGCRequested() const { return minorGCTriggerReason != JS::gcreason::NO_REASON; } + bool majorGCRequested() const { return majorGCTriggerReason != JS::gcreason::NO_REASON; } + bool isGcNeeded() { return minorGCRequested() || majorGCRequested(); } + + bool fullGCForAtomsRequested() const { return fullGCForAtomsRequested_; } + + double computeHeapGrowthFactor(size_t lastBytes); + size_t computeTriggerBytes(double growthFactor, size_t lastBytes); + + JSGCMode gcMode() const { return mode; } + void setGCMode(JSGCMode m) { + mode = m; + marker.setGCMode(mode); + } + + inline void updateOnFreeArenaAlloc(const ChunkInfo& info); + inline void updateOnArenaFree(const ChunkInfo& info); + + ChunkPool& fullChunks(const AutoLockGC& lock) { return fullChunks_; } + ChunkPool& availableChunks(const AutoLockGC& lock) { return availableChunks_; } + ChunkPool& emptyChunks(const AutoLockGC& lock) { return emptyChunks_; } + const ChunkPool& fullChunks(const AutoLockGC& lock) const { return fullChunks_; } + const ChunkPool& availableChunks(const AutoLockGC& lock) const { return availableChunks_; } + const ChunkPool& emptyChunks(const AutoLockGC& lock) const { return emptyChunks_; } + typedef ChainedIter<Chunk*, ChunkPool::Iter, ChunkPool::Iter> NonEmptyChunksIter; + NonEmptyChunksIter allNonEmptyChunks() { + return NonEmptyChunksIter(ChunkPool::Iter(availableChunks_), ChunkPool::Iter(fullChunks_)); + } + + Chunk* getOrAllocChunk(const AutoLockGC& lock, + AutoMaybeStartBackgroundAllocation& maybeStartBGAlloc); + void recycleChunk(Chunk* chunk, const AutoLockGC& lock); + +#ifdef JS_GC_ZEAL + void startVerifyPreBarriers(); + void endVerifyPreBarriers(); + void finishVerifier(); + bool isVerifyPreBarriersEnabled() const { return !!verifyPreData; } +#else + bool isVerifyPreBarriersEnabled() const { return false; } +#endif + + // Free certain LifoAlloc blocks when it is safe to do so. + void freeUnusedLifoBlocksAfterSweeping(LifoAlloc* lifo); + void freeAllLifoBlocksAfterSweeping(LifoAlloc* lifo); + void freeAllLifoBlocksAfterMinorGC(LifoAlloc* lifo); + + // Queue a thunk to run after the next minor GC. + void callAfterMinorGC(void (*thunk)(void* data), void* data) { + nursery.queueSweepAction(thunk, data); + } + + // Public here for ReleaseArenaLists and FinalizeTypedArenas. + void releaseArena(Arena* arena, const AutoLockGC& lock); + + void releaseHeldRelocatedArenas(); + void releaseHeldRelocatedArenasWithoutUnlocking(const AutoLockGC& lock); + + // Allocator + template <AllowGC allowGC> + MOZ_MUST_USE bool checkAllocatorState(JSContext* cx, AllocKind kind); + template <AllowGC allowGC> + JSObject* tryNewNurseryObject(JSContext* cx, size_t thingSize, size_t nDynamicSlots, + const Class* clasp); + template <AllowGC allowGC> + static JSObject* tryNewTenuredObject(ExclusiveContext* cx, AllocKind kind, size_t thingSize, + size_t nDynamicSlots); + template <typename T, AllowGC allowGC> + static T* tryNewTenuredThing(ExclusiveContext* cx, AllocKind kind, size_t thingSize); + static TenuredCell* refillFreeListInGC(Zone* zone, AllocKind thingKind); + + private: + enum IncrementalProgress + { + NotFinished = 0, + Finished + }; + + // For ArenaLists::allocateFromArena() + friend class ArenaLists; + Chunk* pickChunk(const AutoLockGC& lock, + AutoMaybeStartBackgroundAllocation& maybeStartBGAlloc); + Arena* allocateArena(Chunk* chunk, Zone* zone, AllocKind kind, + ShouldCheckThresholds checkThresholds, const AutoLockGC& lock); + void arenaAllocatedDuringGC(JS::Zone* zone, Arena* arena); + + // Allocator internals + MOZ_MUST_USE bool gcIfNeededPerAllocation(JSContext* cx); + template <typename T> + static void checkIncrementalZoneState(ExclusiveContext* cx, T* t); + static TenuredCell* refillFreeListFromAnyThread(ExclusiveContext* cx, AllocKind thingKind, + size_t thingSize); + static TenuredCell* refillFreeListFromMainThread(JSContext* cx, AllocKind thingKind, + size_t thingSize); + static TenuredCell* refillFreeListOffMainThread(ExclusiveContext* cx, AllocKind thingKind); + + /* + * Return the list of chunks that can be released outside the GC lock. + * Must be called either during the GC or with the GC lock taken. + */ + friend class BackgroundDecommitTask; + ChunkPool expireEmptyChunkPool(const AutoLockGC& lock); + void freeEmptyChunks(JSRuntime* rt, const AutoLockGC& lock); + void prepareToFreeChunk(ChunkInfo& info); + + friend class BackgroundAllocTask; + friend class AutoMaybeStartBackgroundAllocation; + bool wantBackgroundAllocation(const AutoLockGC& lock) const; + void startBackgroundAllocTaskIfIdle(); + + void requestMajorGC(JS::gcreason::Reason reason); + SliceBudget defaultBudget(JS::gcreason::Reason reason, int64_t millis); + void budgetIncrementalGC(SliceBudget& budget, AutoLockForExclusiveAccess& lock); + void resetIncrementalGC(AbortReason reason, AutoLockForExclusiveAccess& lock); + + // Assert if the system state is such that we should never + // receive a request to do GC work. + void checkCanCallAPI(); + + // Check if the system state is such that GC has been supressed + // or otherwise delayed. + MOZ_MUST_USE bool checkIfGCAllowedInCurrentState(JS::gcreason::Reason reason); + + gcstats::ZoneGCStats scanZonesBeforeGC(); + void collect(bool nonincrementalByAPI, SliceBudget budget, JS::gcreason::Reason reason) JS_HAZ_GC_CALL; + MOZ_MUST_USE bool gcCycle(bool nonincrementalByAPI, SliceBudget& budget, + JS::gcreason::Reason reason); + void incrementalCollectSlice(SliceBudget& budget, JS::gcreason::Reason reason, + AutoLockForExclusiveAccess& lock); + + void pushZealSelectedObjects(); + void purgeRuntime(AutoLockForExclusiveAccess& lock); + MOZ_MUST_USE bool beginMarkPhase(JS::gcreason::Reason reason, AutoLockForExclusiveAccess& lock); + bool shouldPreserveJITCode(JSCompartment* comp, int64_t currentTime, + JS::gcreason::Reason reason, bool canAllocateMoreCode); + void traceRuntimeForMajorGC(JSTracer* trc, AutoLockForExclusiveAccess& lock); + void traceRuntimeAtoms(JSTracer* trc, AutoLockForExclusiveAccess& lock); + void traceRuntimeCommon(JSTracer* trc, TraceOrMarkRuntime traceOrMark, + AutoLockForExclusiveAccess& lock); + void bufferGrayRoots(); + void maybeDoCycleCollection(); + void markCompartments(); + IncrementalProgress drainMarkStack(SliceBudget& sliceBudget, gcstats::Phase phase); + template <class CompartmentIterT> void markWeakReferences(gcstats::Phase phase); + void markWeakReferencesInCurrentGroup(gcstats::Phase phase); + template <class ZoneIterT, class CompartmentIterT> void markGrayReferences(gcstats::Phase phase); + void markBufferedGrayRoots(JS::Zone* zone); + void markGrayReferencesInCurrentGroup(gcstats::Phase phase); + void markAllWeakReferences(gcstats::Phase phase); + void markAllGrayReferences(gcstats::Phase phase); + + void beginSweepPhase(bool lastGC, AutoLockForExclusiveAccess& lock); + void findZoneGroups(AutoLockForExclusiveAccess& lock); + MOZ_MUST_USE bool findInterZoneEdges(); + void getNextZoneGroup(); + void endMarkingZoneGroup(); + void beginSweepingZoneGroup(AutoLockForExclusiveAccess& lock); + bool shouldReleaseObservedTypes(); + void endSweepingZoneGroup(); + IncrementalProgress sweepPhase(SliceBudget& sliceBudget, AutoLockForExclusiveAccess& lock); + void endSweepPhase(bool lastGC, AutoLockForExclusiveAccess& lock); + void sweepZones(FreeOp* fop, bool lastGC); + void decommitAllWithoutUnlocking(const AutoLockGC& lock); + void startDecommit(); + void queueZonesForBackgroundSweep(ZoneList& zones); + void sweepBackgroundThings(ZoneList& zones, LifoAlloc& freeBlocks); + void assertBackgroundSweepingFinished(); + bool shouldCompact(); + void beginCompactPhase(); + IncrementalProgress compactPhase(JS::gcreason::Reason reason, SliceBudget& sliceBudget, + AutoLockForExclusiveAccess& lock); + void endCompactPhase(JS::gcreason::Reason reason); + void sweepTypesAfterCompacting(Zone* zone); + void sweepZoneAfterCompacting(Zone* zone); + MOZ_MUST_USE bool relocateArenas(Zone* zone, JS::gcreason::Reason reason, + Arena*& relocatedListOut, SliceBudget& sliceBudget); + void updateTypeDescrObjects(MovingTracer* trc, Zone* zone); + void updateCellPointers(MovingTracer* trc, Zone* zone, AllocKinds kinds, size_t bgTaskCount); + void updateAllCellPointers(MovingTracer* trc, Zone* zone); + void updatePointersToRelocatedCells(Zone* zone, AutoLockForExclusiveAccess& lock); + void protectAndHoldArenas(Arena* arenaList); + void unprotectHeldRelocatedArenas(); + void releaseRelocatedArenas(Arena* arenaList); + void releaseRelocatedArenasWithoutUnlocking(Arena* arenaList, const AutoLockGC& lock); + void finishCollection(JS::gcreason::Reason reason); + + void computeNonIncrementalMarkingForValidation(AutoLockForExclusiveAccess& lock); + void validateIncrementalMarking(); + void finishMarkingValidation(); + +#ifdef DEBUG + void checkForCompartmentMismatches(); +#endif + + void callFinalizeCallbacks(FreeOp* fop, JSFinalizeStatus status) const; + void callWeakPointerZoneGroupCallbacks() const; + void callWeakPointerCompartmentCallbacks(JSCompartment* comp) const; + + public: + JSRuntime* rt; + + /* Embedders can use this zone however they wish. */ + JS::Zone* systemZone; + + /* List of compartments and zones (protected by the GC lock). */ + ZoneVector zones; + + Nursery nursery; + StoreBuffer storeBuffer; + + gcstats::Statistics stats; + + GCMarker marker; + + /* Track heap usage for this runtime. */ + HeapUsage usage; + + /* GC scheduling state and parameters. */ + GCSchedulingTunables tunables; + GCSchedulingState schedulingState; + + MemProfiler mMemProfiler; + + private: + // When empty, chunks reside in the emptyChunks pool and are re-used as + // needed or eventually expired if not re-used. The emptyChunks pool gets + // refilled from the background allocation task heuristically so that empty + // chunks should always available for immediate allocation without syscalls. + ChunkPool emptyChunks_; + + // Chunks which have had some, but not all, of their arenas allocated live + // in the available chunk lists. When all available arenas in a chunk have + // been allocated, the chunk is removed from the available list and moved + // to the fullChunks pool. During a GC, if all arenas are free, the chunk + // is moved back to the emptyChunks pool and scheduled for eventual + // release. + ChunkPool availableChunks_; + + // When all arenas in a chunk are used, it is moved to the fullChunks pool + // so as to reduce the cost of operations on the available lists. + ChunkPool fullChunks_; + + RootedValueMap rootsHash; + + size_t maxMallocBytes; + + // An incrementing id used to assign unique ids to cells that require one. + mozilla::Atomic<uint64_t, mozilla::ReleaseAcquire> nextCellUniqueId_; + + /* + * Number of the committed arenas in all GC chunks including empty chunks. + */ + mozilla::Atomic<uint32_t, mozilla::ReleaseAcquire> numArenasFreeCommitted; + VerifyPreTracer* verifyPreData; + + private: + bool chunkAllocationSinceLastGC; + int64_t lastGCTime; + + JSGCMode mode; + + mozilla::Atomic<size_t, mozilla::ReleaseAcquire> numActiveZoneIters; + + /* During shutdown, the GC needs to clean up every possible object. */ + bool cleanUpEverything; + + // Gray marking must be done after all black marking is complete. However, + // we do not have write barriers on XPConnect roots. Therefore, XPConnect + // roots must be accumulated in the first slice of incremental GC. We + // accumulate these roots in each zone's gcGrayRoots vector and then mark + // them later, after black marking is complete for each compartment. This + // accumulation can fail, but in that case we switch to non-incremental GC. + enum class GrayBufferState { + Unused, + Okay, + Failed + }; + GrayBufferState grayBufferState; + bool hasBufferedGrayRoots() const { return grayBufferState == GrayBufferState::Okay; } + + // Clear each zone's gray buffers, but do not change the current state. + void resetBufferedGrayRoots() const; + + // Reset the gray buffering state to Unused. + void clearBufferedGrayRoots() { + grayBufferState = GrayBufferState::Unused; + resetBufferedGrayRoots(); + } + + mozilla::Atomic<JS::gcreason::Reason, mozilla::Relaxed> majorGCTriggerReason; + + JS::gcreason::Reason minorGCTriggerReason; + + /* Perform full GC if rt->keepAtoms() becomes false. */ + bool fullGCForAtomsRequested_; + + /* Incremented at the start of every minor GC. */ + uint64_t minorGCNumber; + + /* Incremented at the start of every major GC. */ + uint64_t majorGCNumber; + + /* The major GC number at which to release observed type information. */ + uint64_t jitReleaseNumber; + + /* Incremented on every GC slice. */ + uint64_t number; + + /* The number at the time of the most recent GC's first slice. */ + uint64_t startNumber; + + /* Whether the currently running GC can finish in multiple slices. */ + bool isIncremental; + + /* Whether all zones are being collected in first GC slice. */ + bool isFull; + + /* Whether the heap will be compacted at the end of GC. */ + bool isCompacting; + + /* The invocation kind of the current GC, taken from the first slice. */ + JSGCInvocationKind invocationKind; + + /* The initial GC reason, taken from the first slice. */ + JS::gcreason::Reason initialReason; + +#ifdef DEBUG + /* + * If this is 0, all cross-compartment proxies must be registered in the + * wrapper map. This checking must be disabled temporarily while creating + * new wrappers. When non-zero, this records the recursion depth of wrapper + * creation. + */ + uintptr_t disableStrictProxyCheckingCount; +#endif + + /* + * The current incremental GC phase. This is also used internally in + * non-incremental GC. + */ + State incrementalState; + + /* Indicates that the last incremental slice exhausted the mark stack. */ + bool lastMarkSlice; + + /* Whether any sweeping will take place in the separate GC helper thread. */ + bool sweepOnBackgroundThread; + + /* Whether observed type information is being released in the current GC. */ + bool releaseObservedTypes; + + /* Whether any black->gray edges were found during marking. */ + BlackGrayEdgeVector foundBlackGrayEdges; + + /* Singly linekd list of zones to be swept in the background. */ + ZoneList backgroundSweepZones; + + /* + * Free LIFO blocks are transferred to this allocator before being freed on + * the background GC thread after sweeping. + */ + LifoAlloc blocksToFreeAfterSweeping; + + /* + * Free LIFO blocks are transferred to this allocator before being freed + * after minor GC. + */ + LifoAlloc blocksToFreeAfterMinorGC; + + /* Index of current zone group (for stats). */ + unsigned zoneGroupIndex; + + /* + * Incremental sweep state. + */ + JS::Zone* zoneGroups; + JS::Zone* currentZoneGroup; + bool sweepingTypes; + unsigned finalizePhase; + JS::Zone* sweepZone; + AllocKind sweepKind; + bool abortSweepAfterCurrentGroup; + + /* + * Concurrent sweep infrastructure. + */ + void startTask(GCParallelTask& task, gcstats::Phase phase, AutoLockHelperThreadState& locked); + void joinTask(GCParallelTask& task, gcstats::Phase phase, AutoLockHelperThreadState& locked); + + /* + * List head of arenas allocated during the sweep phase. + */ + Arena* arenasAllocatedDuringSweep; + + /* + * Incremental compacting state. + */ + bool startedCompacting; + ZoneList zonesToMaybeCompact; + Arena* relocatedArenasToRelease; + +#ifdef JS_GC_ZEAL + MarkingValidator* markingValidator; +#endif + + /* + * Indicates that a GC slice has taken place in the middle of an animation + * frame, rather than at the beginning. In this case, the next slice will be + * delayed so that we don't get back-to-back slices. + */ + bool interFrameGC; + + /* Default budget for incremental GC slice. See js/SliceBudget.h. */ + int64_t defaultTimeBudget_; + + /* + * We disable incremental GC if we encounter a Class with a trace hook + * that does not implement write barriers. + */ + bool incrementalAllowed; + + /* + * GGC can be enabled from the command line while testing. + */ + unsigned generationalDisabled; + + /* + * Whether compacting GC can is enabled globally. + */ + bool compactingEnabled; + + /* + * Some code cannot tolerate compacting GC so it can be disabled temporarily + * with AutoDisableCompactingGC which uses this counter. + */ + unsigned compactingDisabledCount; + + /* + * This is true if we are in the middle of a brain transplant (e.g., + * JS_TransplantObject) or some other operation that can manipulate + * dead zones. + */ + bool manipulatingDeadZones; + + /* + * This field is incremented each time we mark an object inside a + * zone with no incoming cross-compartment pointers. Typically if + * this happens it signals that an incremental GC is marking too much + * stuff. At various times we check this counter and, if it has changed, we + * run an immediate, non-incremental GC to clean up the dead + * zones. This should happen very rarely. + */ + unsigned objectsMarkedInDeadZones; + + bool poked; + + /* + * These options control the zealousness of the GC. At every allocation, + * nextScheduled is decremented. When it reaches zero we do a full GC. + * + * At this point, if zeal_ is one of the types that trigger periodic + * collection, then nextScheduled is reset to the value of zealFrequency. + * Otherwise, no additional GCs take place. + * + * You can control these values in several ways: + * - Set the JS_GC_ZEAL environment variable + * - Call gczeal() or schedulegc() from inside shell-executed JS code + * (see the help for details) + * + * If gcZeal_ == 1 then we perform GCs in select places (during MaybeGC and + * whenever a GC poke happens). This option is mainly useful to embedders. + * + * We use zeal_ == 4 to enable write barrier verification. See the comment + * in jsgc.cpp for more information about this. + * + * zeal_ values from 8 to 10 periodically run different types of + * incremental GC. + * + * zeal_ value 14 performs periodic shrinking collections. + */ +#ifdef JS_GC_ZEAL + uint32_t zealModeBits; + int zealFrequency; + int nextScheduled; + bool deterministicOnly; + int incrementalLimit; + + Vector<JSObject*, 0, SystemAllocPolicy> selectedForMarking; +#endif + + bool fullCompartmentChecks; + + Callback<JSGCCallback> gcCallback; + Callback<JS::DoCycleCollectionCallback> gcDoCycleCollectionCallback; + Callback<JSObjectsTenuredCallback> tenuredCallback; + CallbackVector<JSFinalizeCallback> finalizeCallbacks; + CallbackVector<JSWeakPointerZoneGroupCallback> updateWeakPointerZoneGroupCallbacks; + CallbackVector<JSWeakPointerCompartmentCallback> updateWeakPointerCompartmentCallbacks; + + /* + * Malloc counter to measure memory pressure for GC scheduling. It runs + * from maxMallocBytes down to zero. + */ + mozilla::Atomic<ptrdiff_t, mozilla::ReleaseAcquire> mallocBytesUntilGC; + + /* + * Whether a GC has been triggered as a result of mallocBytesUntilGC + * falling below zero. + */ + mozilla::Atomic<bool, mozilla::ReleaseAcquire> mallocGCTriggered; + + /* + * The trace operations to trace embedding-specific GC roots. One is for + * tracing through black roots and the other is for tracing through gray + * roots. The black/gray distinction is only relevant to the cycle + * collector. + */ + CallbackVector<JSTraceDataOp> blackRootTracers; + Callback<JSTraceDataOp> grayRootTracer; + + /* Always preserve JIT code during GCs, for testing. */ + bool alwaysPreserveCode; + + /* + * Some regions of code are hard for the static rooting hazard analysis to + * understand. In those cases, we trade the static analysis for a dynamic + * analysis. When this is non-zero, we should assert if we trigger, or + * might trigger, a GC. + */ + int inUnsafeRegion; + +#ifdef DEBUG + size_t noGCOrAllocationCheck; + size_t noNurseryAllocationCheck; + + bool arenasEmptyAtShutdown; +#endif + + /* Synchronize GC heap access between main thread and GCHelperState. */ + friend class js::AutoLockGC; + js::Mutex lock; + + BackgroundAllocTask allocTask; + BackgroundDecommitTask decommitTask; + GCHelperState helperState; + + /* + * During incremental sweeping, this field temporarily holds the arenas of + * the current AllocKind being swept in order of increasing free space. + */ + SortedArenaList incrementalSweepList; + + friend class js::GCHelperState; + friend class MarkingValidator; + friend class AutoTraceSession; + friend class AutoEnterIteration; +}; + +/* Prevent compartments and zones from being collected during iteration. */ +class MOZ_RAII AutoEnterIteration { + GCRuntime* gc; + + public: + explicit AutoEnterIteration(GCRuntime* gc_) : gc(gc_) { + ++gc->numActiveZoneIters; + } + + ~AutoEnterIteration() { + MOZ_ASSERT(gc->numActiveZoneIters); + --gc->numActiveZoneIters; + } +}; + +// After pulling a Chunk out of the empty chunks pool, we want to run the +// background allocator to refill it. The code that takes Chunks does so under +// the GC lock. We need to start the background allocation under the helper +// threads lock. To avoid lock inversion we have to delay the start until after +// we are outside the GC lock. This class handles that delay automatically. +class MOZ_RAII AutoMaybeStartBackgroundAllocation +{ + GCRuntime* gc; + + public: + AutoMaybeStartBackgroundAllocation() + : gc(nullptr) + {} + + void tryToStartBackgroundAllocation(GCRuntime& gc) { + this->gc = &gc; + } + + ~AutoMaybeStartBackgroundAllocation() { + if (gc) + gc->startBackgroundAllocTaskIfIdle(); + } +}; + +#ifdef JS_GC_ZEAL + +inline bool +GCRuntime::hasZealMode(ZealMode mode) +{ + static_assert(size_t(ZealMode::Limit) < sizeof(zealModeBits) * 8, + "Zeal modes must fit in zealModeBits"); + return zealModeBits & (1 << uint32_t(mode)); +} + +inline void +GCRuntime::clearZealMode(ZealMode mode) +{ + zealModeBits &= ~(1 << uint32_t(mode)); + MOZ_ASSERT(!hasZealMode(mode)); +} + +inline bool +GCRuntime::upcomingZealousGC() { + return nextScheduled == 1; +} + +inline bool +GCRuntime::needZealousGC() { + if (nextScheduled > 0 && --nextScheduled == 0) { + if (hasZealMode(ZealMode::Alloc) || + hasZealMode(ZealMode::GenerationalGC) || + hasZealMode(ZealMode::IncrementalRootsThenFinish) || + hasZealMode(ZealMode::IncrementalMarkAllThenFinish) || + hasZealMode(ZealMode::IncrementalMultipleSlices) || + hasZealMode(ZealMode::Compact)) + { + nextScheduled = zealFrequency; + } + return true; + } + return false; +} +#else +inline bool GCRuntime::hasZealMode(ZealMode mode) { return false; } +inline void GCRuntime::clearZealMode(ZealMode mode) { } +inline bool GCRuntime::upcomingZealousGC() { return false; } +inline bool GCRuntime::needZealousGC() { return false; } +#endif + +} /* namespace gc */ + +} /* namespace js */ + +#endif diff --git a/js/src/gc/GCTrace.cpp b/js/src/gc/GCTrace.cpp new file mode 100644 index 000000000..5a336b093 --- /dev/null +++ b/js/src/gc/GCTrace.cpp @@ -0,0 +1,243 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifdef JS_GC_TRACE + +#include "gc/GCTrace.h" + +#include <stdio.h> +#include <string.h> + +#include "gc/GCTraceFormat.h" + +#include "js/HashTable.h" + +using namespace js; +using namespace js::gc; + +JS_STATIC_ASSERT(AllocKinds == unsigned(AllocKind::LIMIT)); +JS_STATIC_ASSERT(LastObjectAllocKind == unsigned(AllocKind::OBJECT_LAST)); + +static FILE* gcTraceFile = nullptr; + +static HashSet<const Class*, DefaultHasher<const Class*>, SystemAllocPolicy> tracedClasses; +static HashSet<const ObjectGroup*, DefaultHasher<const ObjectGroup*>, SystemAllocPolicy> tracedGroups; + +static inline void +WriteWord(uint64_t data) +{ + if (gcTraceFile) + fwrite(&data, sizeof(data), 1, gcTraceFile); +} + +static inline void +TraceEvent(GCTraceEvent event, uint64_t payload = 0, uint8_t extra = 0) +{ + MOZ_ASSERT(event < GCTraceEventCount); + MOZ_ASSERT((payload >> TracePayloadBits) == 0); + WriteWord((uint64_t(event) << TraceEventShift) | + (uint64_t(extra) << TraceExtraShift) | payload); +} + +static inline void +TraceAddress(const void* p) +{ + TraceEvent(TraceDataAddress, uint64_t(p)); +} + +static inline void +TraceInt(uint32_t data) +{ + TraceEvent(TraceDataInt, data); +} + +static void +TraceString(const char* string) +{ + JS_STATIC_ASSERT(sizeof(char) == 1); + + size_t length = strlen(string); + const unsigned charsPerWord = sizeof(uint64_t); + unsigned wordCount = (length + charsPerWord - 1) / charsPerWord; + + TraceEvent(TraceDataString, length); + for (unsigned i = 0; i < wordCount; ++i) { + union + { + uint64_t word; + char chars[charsPerWord]; + } data; + strncpy(data.chars, string + (i * charsPerWord), charsPerWord); + WriteWord(data.word); + } +} + +bool +js::gc::InitTrace(GCRuntime& gc) +{ + /* This currently does not support multiple runtimes. */ + MOZ_ALWAYS_TRUE(!gcTraceFile); + + char* filename = getenv("JS_GC_TRACE"); + if (!filename) + return true; + + if (!tracedClasses.init() || !tracedTypes.init()) { + FinishTrace(); + return false; + } + + gcTraceFile = fopen(filename, "w"); + if (!gcTraceFile) { + FinishTrace(); + return false; + } + + TraceEvent(TraceEventInit, 0, TraceFormatVersion); + + /* Trace information about thing sizes. */ + for (auto kind : AllAllocKinds()) + TraceEvent(TraceEventThingSize, Arena::thingSize(kind)); + + return true; +} + +void +js::gc::FinishTrace() +{ + if (gcTraceFile) { + fclose(gcTraceFile); + gcTraceFile = nullptr; + } + tracedClasses.finish(); + tracedTypes.finish(); +} + +bool +js::gc::TraceEnabled() +{ + return gcTraceFile != nullptr; +} + +void +js::gc::TraceNurseryAlloc(Cell* thing, size_t size) +{ + if (thing) { + /* We don't have AllocKind here, but we can work it out from size. */ + unsigned slots = (size - sizeof(JSObject)) / sizeof(JS::Value); + AllocKind kind = GetBackgroundAllocKind(GetGCObjectKind(slots)); + TraceEvent(TraceEventNurseryAlloc, uint64_t(thing), kind); + } +} + +void +js::gc::TraceTenuredAlloc(Cell* thing, AllocKind kind) +{ + if (thing) + TraceEvent(TraceEventTenuredAlloc, uint64_t(thing), kind); +} + +static void +MaybeTraceClass(const Class* clasp) +{ + if (tracedClasses.has(clasp)) + return; + + TraceEvent(TraceEventClassInfo, uint64_t(clasp)); + TraceString(clasp->name); + TraceInt(clasp->flags); + TraceInt(clasp->finalize != nullptr); + + MOZ_ALWAYS_TRUE(tracedClasses.put(clasp)); +} + +static void +MaybeTraceGroup(ObjectGroup* group) +{ + if (tracedGroups.has(group)) + return; + + MaybeTraceClass(group->clasp()); + TraceEvent(TraceEventGroupInfo, uint64_t(group)); + TraceAddress(group->clasp()); + TraceInt(group->flags()); + + MOZ_ALWAYS_TRUE(tracedGroups.put(group)); +} + +void +js::gc::TraceTypeNewScript(ObjectGroup* group) +{ + const size_t bufLength = 128; + static char buffer[bufLength]; + MOZ_ASSERT(group->hasNewScript()); + JSAtom* funName = group->newScript()->fun->displayAtom(); + if (!funName) + return; + + size_t length = funName->length(); + MOZ_ALWAYS_TRUE(length < bufLength); + CopyChars(reinterpret_cast<Latin1Char*>(buffer), *funName); + buffer[length] = 0; + + TraceEvent(TraceEventTypeNewScript, uint64_t(group)); + TraceString(buffer); +} + +void +js::gc::TraceCreateObject(JSObject* object) +{ + if (!gcTraceFile) + return; + + ObjectGroup* group = object->group(); + MaybeTraceGroup(group); + TraceEvent(TraceEventCreateObject, uint64_t(object)); + TraceAddress(group); +} + +void +js::gc::TraceMinorGCStart() +{ + TraceEvent(TraceEventMinorGCStart); +} + +void +js::gc::TracePromoteToTenured(Cell* src, Cell* dst) +{ + TraceEvent(TraceEventPromoteToTenured, uint64_t(src)); + TraceAddress(dst); +} + +void +js::gc::TraceMinorGCEnd() +{ + TraceEvent(TraceEventMinorGCEnd); +} + +void +js::gc::TraceMajorGCStart() +{ + TraceEvent(TraceEventMajorGCStart); +} + +void +js::gc::TraceTenuredFinalize(Cell* thing) +{ + if (!gcTraceFile) + return; + if (thing->tenuredGetAllocKind() == AllocKind::OBJECT_GROUP) + tracedGroups.remove(static_cast<const ObjectGroup*>(thing)); + TraceEvent(TraceEventTenuredFinalize, uint64_t(thing)); +} + +void +js::gc::TraceMajorGCEnd() +{ + TraceEvent(TraceEventMajorGCEnd); +} + +#endif diff --git a/js/src/gc/GCTrace.h b/js/src/gc/GCTrace.h new file mode 100644 index 000000000..dc3586dd9 --- /dev/null +++ b/js/src/gc/GCTrace.h @@ -0,0 +1,55 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_GCTrace_h +#define gc_GCTrace_h + +#include "gc/Heap.h" + +namespace js { + +class ObjectGroup; + +namespace gc { + +#ifdef JS_GC_TRACE + +extern MOZ_MUST_USE bool InitTrace(GCRuntime& gc); +extern void FinishTrace(); +extern bool TraceEnabled(); +extern void TraceNurseryAlloc(Cell* thing, size_t size); +extern void TraceTenuredAlloc(Cell* thing, AllocKind kind); +extern void TraceCreateObject(JSObject* object); +extern void TraceMinorGCStart(); +extern void TracePromoteToTenured(Cell* src, Cell* dst); +extern void TraceMinorGCEnd(); +extern void TraceMajorGCStart(); +extern void TraceTenuredFinalize(Cell* thing); +extern void TraceMajorGCEnd(); +extern void TraceTypeNewScript(js::ObjectGroup* group); + +#else + +inline MOZ_MUST_USE bool InitTrace(GCRuntime& gc) { return true; } +inline void FinishTrace() {} +inline bool TraceEnabled() { return false; } +inline void TraceNurseryAlloc(Cell* thing, size_t size) {} +inline void TraceTenuredAlloc(Cell* thing, AllocKind kind) {} +inline void TraceCreateObject(JSObject* object) {} +inline void TraceMinorGCStart() {} +inline void TracePromoteToTenured(Cell* src, Cell* dst) {} +inline void TraceMinorGCEnd() {} +inline void TraceMajorGCStart() {} +inline void TraceTenuredFinalize(Cell* thing) {} +inline void TraceMajorGCEnd() {} +inline void TraceTypeNewScript(js::ObjectGroup* group) {} + +#endif + +} /* namespace gc */ +} /* namespace js */ + +#endif diff --git a/js/src/gc/GCTraceFormat.h b/js/src/gc/GCTraceFormat.h new file mode 100644 index 000000000..82998b37d --- /dev/null +++ b/js/src/gc/GCTraceFormat.h @@ -0,0 +1,56 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_GCTraceFormat_h +#define gc_GCTraceFormat_h + +/* + * Each trace is stored as a 64-bit word with the following format: + * + * 56 48 0 + * +----------------+----------------+-----------------------------------------+ + * | Event type | Optional extra | Optional payload | + * +----------------+----------------+-----------------------------------------+ + */ + +enum GCTraceEvent { + // Events + TraceEventInit, + TraceEventThingSize, + TraceEventNurseryAlloc, + TraceEventTenuredAlloc, + TraceEventClassInfo, + TraceEventTypeInfo, + TraceEventTypeNewScript, + TraceEventCreateObject, + TraceEventMinorGCStart, + TraceEventPromoteToTenured, + TraceEventMinorGCEnd, + TraceEventMajorGCStart, + TraceEventTenuredFinalize, + TraceEventMajorGCEnd, + + TraceDataAddress, // following TraceEventPromote + TraceDataInt, // following TraceEventClassInfo + TraceDataString, // following TraceEventClassInfo + + GCTraceEventCount +}; + +const unsigned TraceFormatVersion = 1; + +const unsigned TracePayloadBits = 48; + +const unsigned TraceExtraShift = 48; +const unsigned TraceExtraBits = 8; + +const unsigned TraceEventShift = 56; +const unsigned TraceEventBits = 8; + +const unsigned AllocKinds = 22; +const unsigned LastObjectAllocKind = 11; + +#endif diff --git a/js/src/gc/Heap-inl.h b/js/src/gc/Heap-inl.h new file mode 100644 index 000000000..0126e74ae --- /dev/null +++ b/js/src/gc/Heap-inl.h @@ -0,0 +1,29 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_Heap_inl_h +#define gc_Heap_inl_h + +#include "gc/StoreBuffer.h" + +inline void +js::gc::Arena::init(JS::Zone* zoneArg, AllocKind kind) +{ + MOZ_ASSERT(firstFreeSpan.isEmpty()); + MOZ_ASSERT(!zone); + MOZ_ASSERT(!allocated()); + MOZ_ASSERT(!hasDelayedMarking); + MOZ_ASSERT(!allocatedDuringIncremental); + MOZ_ASSERT(!markOverflow); + MOZ_ASSERT(!auxNextLink); + + zone = zoneArg; + allocKind = size_t(kind); + setAsFullyUnused(); + bufferedCells = &ArenaCellSet::Empty; +} + +#endif diff --git a/js/src/gc/Heap.h b/js/src/gc/Heap.h new file mode 100644 index 000000000..2a9390e91 --- /dev/null +++ b/js/src/gc/Heap.h @@ -0,0 +1,1385 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_Heap_h +#define gc_Heap_h + +#include "mozilla/ArrayUtils.h" +#include "mozilla/Atomics.h" +#include "mozilla/Attributes.h" +#include "mozilla/DebugOnly.h" +#include "mozilla/EnumeratedArray.h" +#include "mozilla/EnumeratedRange.h" +#include "mozilla/PodOperations.h" + +#include <stddef.h> +#include <stdint.h> + +#include "jsfriendapi.h" +#include "jspubtd.h" +#include "jstypes.h" +#include "jsutil.h" + +#include "ds/BitArray.h" +#include "gc/Memory.h" +#include "js/GCAPI.h" +#include "js/HeapAPI.h" +#include "js/RootingAPI.h" +#include "js/TracingAPI.h" + +struct JSRuntime; + +namespace JS { +namespace shadow { +struct Runtime; +} // namespace shadow +} // namespace JS + +namespace js { + +class AutoLockGC; +class FreeOp; + +extern bool +RuntimeFromMainThreadIsHeapMajorCollecting(JS::shadow::Zone* shadowZone); + +#ifdef DEBUG + +// Barriers can't be triggered during backend Ion compilation, which may run on +// a helper thread. +extern bool +CurrentThreadIsIonCompiling(); +#endif + +// The return value indicates if anything was unmarked. +extern bool +UnmarkGrayCellRecursively(gc::Cell* cell, JS::TraceKind kind); + +extern void +TraceManuallyBarrieredGenericPointerEdge(JSTracer* trc, gc::Cell** thingp, const char* name); + +namespace gc { + +class Arena; +class ArenaCellSet; +class ArenaList; +class SortedArenaList; +struct Chunk; + +/* + * This flag allows an allocation site to request a specific heap based upon the + * estimated lifetime or lifetime requirements of objects allocated from that + * site. + */ +enum InitialHeap { + DefaultHeap, + TenuredHeap +}; + +/* The GC allocation kinds. */ +// FIXME: uint8_t would make more sense for the underlying type, but causes +// miscompilations in GCC (fixed in 4.8.5 and 4.9.3). See also bug 1143966. +enum class AllocKind { + FIRST, + OBJECT_FIRST = FIRST, + FUNCTION = FIRST, + FUNCTION_EXTENDED, + OBJECT0, + OBJECT0_BACKGROUND, + OBJECT2, + OBJECT2_BACKGROUND, + OBJECT4, + OBJECT4_BACKGROUND, + OBJECT8, + OBJECT8_BACKGROUND, + OBJECT12, + OBJECT12_BACKGROUND, + OBJECT16, + OBJECT16_BACKGROUND, + OBJECT_LIMIT, + OBJECT_LAST = OBJECT_LIMIT - 1, + SCRIPT, + LAZY_SCRIPT, + SHAPE, + ACCESSOR_SHAPE, + BASE_SHAPE, + OBJECT_GROUP, + FAT_INLINE_STRING, + STRING, + EXTERNAL_STRING, + FAT_INLINE_ATOM, + ATOM, + SYMBOL, + JITCODE, + SCOPE, + LIMIT, + LAST = LIMIT - 1 +}; + +// Macro to enumerate the different allocation kinds supplying information about +// the trace kind, C++ type and allocation size. +#define FOR_EACH_OBJECT_ALLOCKIND(D) \ + /* AllocKind TraceKind TypeName SizedType */ \ + D(FUNCTION, Object, JSObject, JSFunction) \ + D(FUNCTION_EXTENDED, Object, JSObject, FunctionExtended) \ + D(OBJECT0, Object, JSObject, JSObject_Slots0) \ + D(OBJECT0_BACKGROUND, Object, JSObject, JSObject_Slots0) \ + D(OBJECT2, Object, JSObject, JSObject_Slots2) \ + D(OBJECT2_BACKGROUND, Object, JSObject, JSObject_Slots2) \ + D(OBJECT4, Object, JSObject, JSObject_Slots4) \ + D(OBJECT4_BACKGROUND, Object, JSObject, JSObject_Slots4) \ + D(OBJECT8, Object, JSObject, JSObject_Slots8) \ + D(OBJECT8_BACKGROUND, Object, JSObject, JSObject_Slots8) \ + D(OBJECT12, Object, JSObject, JSObject_Slots12) \ + D(OBJECT12_BACKGROUND, Object, JSObject, JSObject_Slots12) \ + D(OBJECT16, Object, JSObject, JSObject_Slots16) \ + D(OBJECT16_BACKGROUND, Object, JSObject, JSObject_Slots16) + +#define FOR_EACH_NONOBJECT_ALLOCKIND(D) \ + /* AllocKind TraceKind TypeName SizedType */ \ + D(SCRIPT, Script, JSScript, JSScript) \ + D(LAZY_SCRIPT, LazyScript, js::LazyScript, js::LazyScript) \ + D(SHAPE, Shape, js::Shape, js::Shape) \ + D(ACCESSOR_SHAPE, Shape, js::AccessorShape, js::AccessorShape) \ + D(BASE_SHAPE, BaseShape, js::BaseShape, js::BaseShape) \ + D(OBJECT_GROUP, ObjectGroup, js::ObjectGroup, js::ObjectGroup) \ + D(FAT_INLINE_STRING, String, JSFatInlineString, JSFatInlineString) \ + D(STRING, String, JSString, JSString) \ + D(EXTERNAL_STRING, String, JSExternalString, JSExternalString) \ + D(FAT_INLINE_ATOM, String, js::FatInlineAtom, js::FatInlineAtom) \ + D(ATOM, String, js::NormalAtom, js::NormalAtom) \ + D(SYMBOL, Symbol, JS::Symbol, JS::Symbol) \ + D(JITCODE, JitCode, js::jit::JitCode, js::jit::JitCode) \ + D(SCOPE, Scope, js::Scope, js::Scope) + +#define FOR_EACH_ALLOCKIND(D) \ + FOR_EACH_OBJECT_ALLOCKIND(D) \ + FOR_EACH_NONOBJECT_ALLOCKIND(D) + +static_assert(int(AllocKind::FIRST) == 0, "Various places depend on AllocKind starting at 0, " + "please audit them carefully!"); +static_assert(int(AllocKind::OBJECT_FIRST) == 0, "Various places depend on AllocKind::OBJECT_FIRST " + "being 0, please audit them carefully!"); + +inline bool +IsAllocKind(AllocKind kind) +{ + return kind >= AllocKind::FIRST && kind <= AllocKind::LIMIT; +} + +inline bool +IsValidAllocKind(AllocKind kind) +{ + return kind >= AllocKind::FIRST && kind <= AllocKind::LAST; +} + +inline bool +IsObjectAllocKind(AllocKind kind) +{ + return kind >= AllocKind::OBJECT_FIRST && kind <= AllocKind::OBJECT_LAST; +} + +inline bool +IsShapeAllocKind(AllocKind kind) +{ + return kind == AllocKind::SHAPE || kind == AllocKind::ACCESSOR_SHAPE; +} + +// Returns a sequence for use in a range-based for loop, +// to iterate over all alloc kinds. +inline decltype(mozilla::MakeEnumeratedRange(AllocKind::FIRST, AllocKind::LIMIT)) +AllAllocKinds() +{ + return mozilla::MakeEnumeratedRange(AllocKind::FIRST, AllocKind::LIMIT); +} + +// Returns a sequence for use in a range-based for loop, +// to iterate over all object alloc kinds. +inline decltype(mozilla::MakeEnumeratedRange(AllocKind::OBJECT_FIRST, AllocKind::OBJECT_LIMIT)) +ObjectAllocKinds() +{ + return mozilla::MakeEnumeratedRange(AllocKind::OBJECT_FIRST, AllocKind::OBJECT_LIMIT); +} + +// Returns a sequence for use in a range-based for loop, +// to iterate over alloc kinds from |first| to |limit|, exclusive. +inline decltype(mozilla::MakeEnumeratedRange(AllocKind::FIRST, AllocKind::LIMIT)) +SomeAllocKinds(AllocKind first = AllocKind::FIRST, AllocKind limit = AllocKind::LIMIT) +{ + MOZ_ASSERT(IsAllocKind(first), "|first| is not a valid AllocKind!"); + MOZ_ASSERT(IsAllocKind(limit), "|limit| is not a valid AllocKind!"); + return mozilla::MakeEnumeratedRange(first, limit); +} + +// AllAllocKindArray<ValueType> gives an enumerated array of ValueTypes, +// with each index corresponding to a particular alloc kind. +template<typename ValueType> using AllAllocKindArray = + mozilla::EnumeratedArray<AllocKind, AllocKind::LIMIT, ValueType>; + +// ObjectAllocKindArray<ValueType> gives an enumerated array of ValueTypes, +// with each index corresponding to a particular object alloc kind. +template<typename ValueType> using ObjectAllocKindArray = + mozilla::EnumeratedArray<AllocKind, AllocKind::OBJECT_LIMIT, ValueType>; + +static inline JS::TraceKind +MapAllocToTraceKind(AllocKind kind) +{ + static const JS::TraceKind map[] = { +#define EXPAND_ELEMENT(allocKind, traceKind, type, sizedType) \ + JS::TraceKind::traceKind, +FOR_EACH_ALLOCKIND(EXPAND_ELEMENT) +#undef EXPAND_ELEMENT + }; + + static_assert(MOZ_ARRAY_LENGTH(map) == size_t(AllocKind::LIMIT), + "AllocKind-to-TraceKind mapping must be in sync"); + return map[size_t(kind)]; +} + +/* + * This must be an upper bound, but we do not need the least upper bound, so + * we just exclude non-background objects. + */ +static const size_t MAX_BACKGROUND_FINALIZE_KINDS = + size_t(AllocKind::LIMIT) - size_t(AllocKind::OBJECT_LIMIT) / 2; + +class TenuredCell; + +// A GC cell is the base class for all GC things. +struct Cell +{ + public: + MOZ_ALWAYS_INLINE bool isTenured() const { return !IsInsideNursery(this); } + MOZ_ALWAYS_INLINE const TenuredCell& asTenured() const; + MOZ_ALWAYS_INLINE TenuredCell& asTenured(); + + inline JSRuntime* runtimeFromMainThread() const; + inline JS::shadow::Runtime* shadowRuntimeFromMainThread() const; + + // Note: Unrestricted access to the runtime of a GC thing from an arbitrary + // thread can easily lead to races. Use this method very carefully. + inline JSRuntime* runtimeFromAnyThread() const; + inline JS::shadow::Runtime* shadowRuntimeFromAnyThread() const; + + // May be overridden by GC thing kinds that have a compartment pointer. + inline JSCompartment* maybeCompartment() const { return nullptr; } + + inline StoreBuffer* storeBuffer() const; + + inline JS::TraceKind getTraceKind() const; + + static MOZ_ALWAYS_INLINE bool needWriteBarrierPre(JS::Zone* zone); + +#ifdef DEBUG + inline bool isAligned() const; + void dump(FILE* fp) const; + void dump() const; +#endif + + protected: + inline uintptr_t address() const; + inline Chunk* chunk() const; +} JS_HAZ_GC_THING; + +// A GC TenuredCell gets behaviors that are valid for things in the Tenured +// heap, such as access to the arena and mark bits. +class TenuredCell : public Cell +{ + public: + // Construct a TenuredCell from a void*, making various sanity assertions. + static MOZ_ALWAYS_INLINE TenuredCell* fromPointer(void* ptr); + static MOZ_ALWAYS_INLINE const TenuredCell* fromPointer(const void* ptr); + + // Mark bit management. + MOZ_ALWAYS_INLINE bool isMarked(uint32_t color = BLACK) const; + // The return value indicates if the cell went from unmarked to marked. + MOZ_ALWAYS_INLINE bool markIfUnmarked(uint32_t color = BLACK) const; + MOZ_ALWAYS_INLINE void unmark(uint32_t color) const; + MOZ_ALWAYS_INLINE void copyMarkBitsFrom(const TenuredCell* src); + + // Note: this is in TenuredCell because JSObject subclasses are sometimes + // used tagged. + static MOZ_ALWAYS_INLINE bool isNullLike(const Cell* thing) { return !thing; } + + // Access to the arena. + inline Arena* arena() const; + inline AllocKind getAllocKind() const; + inline JS::TraceKind getTraceKind() const; + inline JS::Zone* zone() const; + inline JS::Zone* zoneFromAnyThread() const; + inline bool isInsideZone(JS::Zone* zone) const; + + MOZ_ALWAYS_INLINE JS::shadow::Zone* shadowZone() const { + return JS::shadow::Zone::asShadowZone(zone()); + } + MOZ_ALWAYS_INLINE JS::shadow::Zone* shadowZoneFromAnyThread() const { + return JS::shadow::Zone::asShadowZone(zoneFromAnyThread()); + } + + static MOZ_ALWAYS_INLINE void readBarrier(TenuredCell* thing); + static MOZ_ALWAYS_INLINE void writeBarrierPre(TenuredCell* thing); + + static MOZ_ALWAYS_INLINE void writeBarrierPost(void* cellp, TenuredCell* prior, + TenuredCell* next); + + // Default implementation for kinds that don't require fixup. + void fixupAfterMovingGC() {} + +#ifdef DEBUG + inline bool isAligned() const; +#endif +}; + +/* Cells are aligned to CellShift, so the largest tagged null pointer is: */ +const uintptr_t LargestTaggedNullCellPointer = (1 << CellShift) - 1; + +constexpr size_t +DivideAndRoundUp(size_t numerator, size_t divisor) { + return (numerator + divisor - 1) / divisor; +} + +const size_t ArenaCellCount = ArenaSize / CellSize; +static_assert(ArenaSize % CellSize == 0, "Arena size must be a multiple of cell size"); + +/* + * The mark bitmap has one bit per each GC cell. For multi-cell GC things this + * wastes space but allows to avoid expensive devisions by thing's size when + * accessing the bitmap. In addition this allows to use some bits for colored + * marking during the cycle GC. + */ +const size_t ArenaBitmapBits = ArenaCellCount; +const size_t ArenaBitmapBytes = DivideAndRoundUp(ArenaBitmapBits, 8); +const size_t ArenaBitmapWords = DivideAndRoundUp(ArenaBitmapBits, JS_BITS_PER_WORD); + +/* + * A FreeSpan represents a contiguous sequence of free cells in an Arena. It + * can take two forms. + * + * - In an empty span, |first| and |last| are both zero. + * + * - In a non-empty span, |first| is the address of the first free thing in the + * span, and |last| is the address of the last free thing in the span. + * Furthermore, the memory pointed to by |last| holds a FreeSpan structure + * that points to the next span (which may be empty); this works because + * sizeof(FreeSpan) is less than the smallest thingSize. + */ +class FreeSpan +{ + friend class Arena; + friend class ArenaCellIterImpl; + + uint16_t first; + uint16_t last; + + public: + // This inits just |first| and |last|; if the span is non-empty it doesn't + // do anything with the next span stored at |last|. + void initBounds(uintptr_t firstArg, uintptr_t lastArg, const Arena* arena) { + checkRange(firstArg, lastArg, arena); + first = firstArg; + last = lastArg; + } + + void initAsEmpty() { + first = 0; + last = 0; + } + + // This sets |first| and |last|, and also sets the next span stored at + // |last| as empty. (As a result, |firstArg| and |lastArg| cannot represent + // an empty span.) + void initFinal(uintptr_t firstArg, uintptr_t lastArg, const Arena* arena) { + initBounds(firstArg, lastArg, arena); + FreeSpan* last = nextSpanUnchecked(arena); + last->initAsEmpty(); + checkSpan(arena); + } + + bool isEmpty() const { + return !first; + } + + Arena* getArenaUnchecked() { return reinterpret_cast<Arena*>(this); } + inline Arena* getArena(); + + static size_t offsetOfFirst() { + return offsetof(FreeSpan, first); + } + + static size_t offsetOfLast() { + return offsetof(FreeSpan, last); + } + + // Like nextSpan(), but no checking of the following span is done. + FreeSpan* nextSpanUnchecked(const Arena* arena) const { + MOZ_ASSERT(arena && !isEmpty()); + return reinterpret_cast<FreeSpan*>(uintptr_t(arena) + last); + } + + const FreeSpan* nextSpan(const Arena* arena) const { + checkSpan(arena); + return nextSpanUnchecked(arena); + } + + MOZ_ALWAYS_INLINE TenuredCell* allocate(size_t thingSize) { + // Eschew the usual checks, because this might be the placeholder span. + // If this is somehow an invalid, non-empty span, checkSpan() will catch it. + Arena* arena = getArenaUnchecked(); + checkSpan(arena); + uintptr_t thing = uintptr_t(arena) + first; + if (first < last) { + // We have space for at least two more things, so do a simple bump-allocate. + first += thingSize; + } else if (MOZ_LIKELY(first)) { + // The last space points to the next free span (which may be empty). + const FreeSpan* next = nextSpan(arena); + first = next->first; + last = next->last; + } else { + return nullptr; // The span is empty. + } + checkSpan(arena); + JS_EXTRA_POISON(reinterpret_cast<void*>(thing), JS_ALLOCATED_TENURED_PATTERN, thingSize); + MemProfiler::SampleTenured(reinterpret_cast<void*>(thing), thingSize); + return reinterpret_cast<TenuredCell*>(thing); + } + + inline void checkSpan(const Arena* arena) const; + inline void checkRange(uintptr_t first, uintptr_t last, const Arena* arena) const; +}; + +/* + * Arenas are the allocation units of the tenured heap in the GC. An arena + * is 4kiB in size and 4kiB-aligned. It starts with several header fields + * followed by some bytes of padding. The remainder of the arena is filled + * with GC things of a particular AllocKind. The padding ensures that the + * GC thing array ends exactly at the end of the arena: + * + * <----------------------------------------------> = ArenaSize bytes + * +---------------+---------+----+----+-----+----+ + * | header fields | padding | T0 | T1 | ... | Tn | + * +---------------+---------+----+----+-----+----+ + * <-------------------------> = first thing offset + */ +class Arena +{ + static JS_FRIEND_DATA(const uint32_t) ThingSizes[]; + static JS_FRIEND_DATA(const uint32_t) FirstThingOffsets[]; + static JS_FRIEND_DATA(const uint32_t) ThingsPerArena[]; + + /* + * The first span of free things in the arena. Most of these spans are + * stored as offsets in free regions of the data array, and most operations + * on FreeSpans take an Arena pointer for safety. However, the FreeSpans + * used for allocation are stored here, at the start of an Arena, and use + * their own address to grab the next span within the same Arena. + */ + FreeSpan firstFreeSpan; + + public: + /* + * The zone that this Arena is contained within, when allocated. The offset + * of this field must match the ArenaZoneOffset stored in js/HeapAPI.h, + * as is statically asserted below. + */ + JS::Zone* zone; + + /* + * Arena::next has two purposes: when unallocated, it points to the next + * available Arena. When allocated, it points to the next Arena in the same + * zone and with the same alloc kind. + */ + Arena* next; + + private: + /* + * One of the AllocKind constants or AllocKind::LIMIT when the arena does + * not contain any GC things and is on the list of empty arenas in the GC + * chunk. + * + * We use 8 bits for the alloc kind so the compiler can use byte-level + * memory instructions to access it. + */ + size_t allocKind : 8; + + public: + /* + * When collecting we sometimes need to keep an auxillary list of arenas, + * for which we use the following fields. This happens for several reasons: + * + * When recursive marking uses too much stack, the marking is delayed and + * the corresponding arenas are put into a stack. To distinguish the bottom + * of the stack from the arenas not present in the stack we use the + * markOverflow flag to tag arenas on the stack. + * + * Delayed marking is also used for arenas that we allocate into during an + * incremental GC. In this case, we intend to mark all the objects in the + * arena, and it's faster to do this marking in bulk. + * + * When sweeping we keep track of which arenas have been allocated since + * the end of the mark phase. This allows us to tell whether a pointer to + * an unmarked object is yet to be finalized or has already been + * reallocated. We set the allocatedDuringIncremental flag for this and + * clear it at the end of the sweep phase. + * + * To minimize the size of the header fields we record the next linkage as + * address() >> ArenaShift and pack it with the allocKind and the flags. + */ + size_t hasDelayedMarking : 1; + size_t allocatedDuringIncremental : 1; + size_t markOverflow : 1; + size_t auxNextLink : JS_BITS_PER_WORD - 8 - 1 - 1 - 1; + static_assert(ArenaShift >= 8 + 1 + 1 + 1, + "Arena::auxNextLink packing assumes that ArenaShift has " + "enough bits to cover allocKind and hasDelayedMarking."); + + /* + * If non-null, points to an ArenaCellSet that represents the set of cells + * in this arena that are in the nursery's store buffer. + */ + ArenaCellSet* bufferedCells; + + /* + * The size of data should be |ArenaSize - offsetof(data)|, but the offset + * is not yet known to the compiler, so we do it by hand. |firstFreeSpan| + * takes up 8 bytes on 64-bit due to alignment requirements; the rest are + * obvious. This constant is stored in js/HeapAPI.h. + */ + uint8_t data[ArenaSize - ArenaHeaderSize]; + + void init(JS::Zone* zoneArg, AllocKind kind); + + // Sets |firstFreeSpan| to the Arena's entire valid range, and + // also sets the next span stored at |firstFreeSpan.last| as empty. + void setAsFullyUnused() { + AllocKind kind = getAllocKind(); + firstFreeSpan.first = firstThingOffset(kind); + firstFreeSpan.last = lastThingOffset(kind); + FreeSpan* last = firstFreeSpan.nextSpanUnchecked(this); + last->initAsEmpty(); + } + + void setAsNotAllocated() { + firstFreeSpan.initAsEmpty(); + zone = nullptr; + allocKind = size_t(AllocKind::LIMIT); + hasDelayedMarking = 0; + allocatedDuringIncremental = 0; + markOverflow = 0; + auxNextLink = 0; + bufferedCells = nullptr; + } + + uintptr_t address() const { + checkAddress(); + return uintptr_t(this); + } + + inline void checkAddress() const; + + inline Chunk* chunk() const; + + bool allocated() const { + MOZ_ASSERT(IsAllocKind(AllocKind(allocKind))); + return IsValidAllocKind(AllocKind(allocKind)); + } + + AllocKind getAllocKind() const { + MOZ_ASSERT(allocated()); + return AllocKind(allocKind); + } + + FreeSpan* getFirstFreeSpan() { return &firstFreeSpan; } + + static size_t thingSize(AllocKind kind) { return ThingSizes[size_t(kind)]; } + static size_t thingsPerArena(AllocKind kind) { return ThingsPerArena[size_t(kind)]; } + static size_t thingsSpan(AllocKind kind) { return thingsPerArena(kind) * thingSize(kind); } + + static size_t firstThingOffset(AllocKind kind) { return FirstThingOffsets[size_t(kind)]; } + static size_t lastThingOffset(AllocKind kind) { return ArenaSize - thingSize(kind); } + + size_t getThingSize() const { return thingSize(getAllocKind()); } + size_t getThingsPerArena() const { return thingsPerArena(getAllocKind()); } + size_t getThingsSpan() const { return getThingsPerArena() * getThingSize(); } + + uintptr_t thingsStart() const { return address() + firstThingOffset(getAllocKind()); } + uintptr_t thingsEnd() const { return address() + ArenaSize; } + + bool isEmpty() const { + // Arena is empty if its first span covers the whole arena. + firstFreeSpan.checkSpan(this); + AllocKind kind = getAllocKind(); + return firstFreeSpan.first == firstThingOffset(kind) && + firstFreeSpan.last == lastThingOffset(kind); + } + + bool hasFreeThings() const { return !firstFreeSpan.isEmpty(); } + + size_t numFreeThings(size_t thingSize) const { + firstFreeSpan.checkSpan(this); + size_t numFree = 0; + const FreeSpan* span = &firstFreeSpan; + for (; !span->isEmpty(); span = span->nextSpan(this)) + numFree += (span->last - span->first) / thingSize + 1; + return numFree; + } + + size_t countFreeCells() { return numFreeThings(getThingSize()); } + size_t countUsedCells() { return getThingsPerArena() - countFreeCells(); } + + bool inFreeList(uintptr_t thing) { + uintptr_t base = address(); + const FreeSpan* span = &firstFreeSpan; + for (; !span->isEmpty(); span = span->nextSpan(this)) { + /* If the thing comes before the current span, it's not free. */ + if (thing < base + span->first) + return false; + + /* If we find it before the end of the span, it's free. */ + if (thing <= base + span->last) + return true; + } + return false; + } + + static bool isAligned(uintptr_t thing, size_t thingSize) { + /* Things ends at the arena end. */ + uintptr_t tailOffset = ArenaSize - (thing & ArenaMask); + return tailOffset % thingSize == 0; + } + + Arena* getNextDelayedMarking() const { + MOZ_ASSERT(hasDelayedMarking); + return reinterpret_cast<Arena*>(auxNextLink << ArenaShift); + } + + void setNextDelayedMarking(Arena* arena) { + MOZ_ASSERT(!(uintptr_t(arena) & ArenaMask)); + MOZ_ASSERT(!auxNextLink && !hasDelayedMarking); + hasDelayedMarking = 1; + if (arena) + auxNextLink = arena->address() >> ArenaShift; + } + + void unsetDelayedMarking() { + MOZ_ASSERT(hasDelayedMarking); + hasDelayedMarking = 0; + auxNextLink = 0; + } + + Arena* getNextAllocDuringSweep() const { + MOZ_ASSERT(allocatedDuringIncremental); + return reinterpret_cast<Arena*>(auxNextLink << ArenaShift); + } + + void setNextAllocDuringSweep(Arena* arena) { + MOZ_ASSERT(!(uintptr_t(arena) & ArenaMask)); + MOZ_ASSERT(!auxNextLink && !allocatedDuringIncremental); + allocatedDuringIncremental = 1; + if (arena) + auxNextLink = arena->address() >> ArenaShift; + } + + void unsetAllocDuringSweep() { + MOZ_ASSERT(allocatedDuringIncremental); + allocatedDuringIncremental = 0; + auxNextLink = 0; + } + + template <typename T> + size_t finalize(FreeOp* fop, AllocKind thingKind, size_t thingSize); + + static void staticAsserts(); + + void unmarkAll(); + + static size_t offsetOfBufferedCells() { + return offsetof(Arena, bufferedCells); + } +}; + +static_assert(ArenaZoneOffset == offsetof(Arena, zone), + "The hardcoded API zone offset must match the actual offset."); + +static_assert(sizeof(Arena) == ArenaSize, + "ArenaSize must match the actual size of the Arena structure."); + +static_assert(offsetof(Arena, data) == ArenaHeaderSize, + "ArenaHeaderSize must match the actual size of the header fields."); + +inline Arena* +FreeSpan::getArena() +{ + Arena* arena = getArenaUnchecked(); + arena->checkAddress(); + return arena; +} + +inline void +FreeSpan::checkSpan(const Arena* arena) const +{ +#ifdef DEBUG + if (!first) { + MOZ_ASSERT(!first && !last); + return; + } + + arena->checkAddress(); + checkRange(first, last, arena); + + // If there's a following span, it must have a higher address, + // and the gap must be at least 2 * thingSize. + const FreeSpan* next = nextSpanUnchecked(arena); + if (next->first) { + checkRange(next->first, next->last, arena); + size_t thingSize = arena->getThingSize(); + MOZ_ASSERT(last + 2 * thingSize <= next->first); + } +#endif +} + +inline void +FreeSpan::checkRange(uintptr_t first, uintptr_t last, const Arena* arena) const +{ +#ifdef DEBUG + MOZ_ASSERT(arena); + MOZ_ASSERT(first <= last); + AllocKind thingKind = arena->getAllocKind(); + MOZ_ASSERT(first >= Arena::firstThingOffset(thingKind)); + MOZ_ASSERT(last <= Arena::lastThingOffset(thingKind)); + MOZ_ASSERT((last - first) % Arena::thingSize(thingKind) == 0); +#endif +} + +/* + * The tail of the chunk info is shared between all chunks in the system, both + * nursery and tenured. This structure is locatable from any GC pointer by + * aligning to 1MiB. + */ +struct ChunkTrailer +{ + /* Construct a Nursery ChunkTrailer. */ + ChunkTrailer(JSRuntime* rt, StoreBuffer* sb) + : location(ChunkLocation::Nursery), storeBuffer(sb), runtime(rt) + {} + + /* Construct a Tenured heap ChunkTrailer. */ + explicit ChunkTrailer(JSRuntime* rt) + : location(ChunkLocation::TenuredHeap), storeBuffer(nullptr), runtime(rt) + {} + + public: + /* The index the chunk in the nursery, or LocationTenuredHeap. */ + ChunkLocation location; + uint32_t padding; + + /* The store buffer for writes to things in this chunk or nullptr. */ + StoreBuffer* storeBuffer; + + /* This provides quick access to the runtime from absolutely anywhere. */ + JSRuntime* runtime; +}; + +static_assert(sizeof(ChunkTrailer) == ChunkTrailerSize, + "ChunkTrailer size must match the API defined size."); + +/* The chunk header (located at the end of the chunk to preserve arena alignment). */ +struct ChunkInfo +{ + void init() { + next = prev = nullptr; + } + + private: + friend class ChunkPool; + Chunk* next; + Chunk* prev; + + public: + /* Free arenas are linked together with arena.next. */ + Arena* freeArenasHead; + +#if JS_BITS_PER_WORD == 32 + /* + * Calculating sizes and offsets is simpler if sizeof(ChunkInfo) is + * architecture-independent. + */ + char padding[24]; +#endif + + /* + * Decommitted arenas are tracked by a bitmap in the chunk header. We use + * this offset to start our search iteration close to a decommitted arena + * that we can allocate. + */ + uint32_t lastDecommittedArenaOffset; + + /* Number of free arenas, either committed or decommitted. */ + uint32_t numArenasFree; + + /* Number of free, committed arenas. */ + uint32_t numArenasFreeCommitted; +}; + +/* + * Calculating ArenasPerChunk: + * + * In order to figure out how many Arenas will fit in a chunk, we need to know + * how much extra space is available after we allocate the header data. This + * is a problem because the header size depends on the number of arenas in the + * chunk. The two dependent fields are bitmap and decommittedArenas. + * + * For the mark bitmap, we know that each arena will use a fixed number of full + * bytes: ArenaBitmapBytes. The full size of the header data is this number + * multiplied by the eventual number of arenas we have in the header. We, + * conceptually, distribute this header data among the individual arenas and do + * not include it in the header. This way we do not have to worry about its + * variable size: it gets attached to the variable number we are computing. + * + * For the decommitted arena bitmap, we only have 1 bit per arena, so this + * technique will not work. Instead, we observe that we do not have enough + * header info to fill 8 full arenas: it is currently 4 on 64bit, less on + * 32bit. Thus, with current numbers, we need 64 bytes for decommittedArenas. + * This will not become 63 bytes unless we double the data required in the + * header. Therefore, we just compute the number of bytes required to track + * every possible arena and do not worry about slop bits, since there are too + * few to usefully allocate. + * + * To actually compute the number of arenas we can allocate in a chunk, we + * divide the amount of available space less the header info (not including + * the mark bitmap which is distributed into the arena size) by the size of + * the arena (with the mark bitmap bytes it uses). + */ +const size_t BytesPerArenaWithHeader = ArenaSize + ArenaBitmapBytes; +const size_t ChunkDecommitBitmapBytes = ChunkSize / ArenaSize / JS_BITS_PER_BYTE; +const size_t ChunkBytesAvailable = ChunkSize - sizeof(ChunkTrailer) - sizeof(ChunkInfo) - ChunkDecommitBitmapBytes; +const size_t ArenasPerChunk = ChunkBytesAvailable / BytesPerArenaWithHeader; + +#ifdef JS_GC_SMALL_CHUNK_SIZE +static_assert(ArenasPerChunk == 62, "Do not accidentally change our heap's density."); +#else +static_assert(ArenasPerChunk == 252, "Do not accidentally change our heap's density."); +#endif + +/* A chunk bitmap contains enough mark bits for all the cells in a chunk. */ +struct ChunkBitmap +{ + volatile uintptr_t bitmap[ArenaBitmapWords * ArenasPerChunk]; + + public: + ChunkBitmap() { } + + MOZ_ALWAYS_INLINE void getMarkWordAndMask(const Cell* cell, uint32_t color, + uintptr_t** wordp, uintptr_t* maskp) + { + detail::GetGCThingMarkWordAndMask(uintptr_t(cell), color, wordp, maskp); + } + + MOZ_ALWAYS_INLINE MOZ_TSAN_BLACKLIST bool isMarked(const Cell* cell, uint32_t color) { + uintptr_t* word, mask; + getMarkWordAndMask(cell, color, &word, &mask); + return *word & mask; + } + + // The return value indicates if the cell went from unmarked to marked. + MOZ_ALWAYS_INLINE bool markIfUnmarked(const Cell* cell, uint32_t color) { + uintptr_t* word, mask; + getMarkWordAndMask(cell, BLACK, &word, &mask); + if (*word & mask) + return false; + *word |= mask; + if (color != BLACK) { + /* + * We use getMarkWordAndMask to recalculate both mask and word as + * doing just mask << color may overflow the mask. + */ + getMarkWordAndMask(cell, color, &word, &mask); + if (*word & mask) + return false; + *word |= mask; + } + return true; + } + + MOZ_ALWAYS_INLINE void unmark(const Cell* cell, uint32_t color) { + uintptr_t* word, mask; + getMarkWordAndMask(cell, color, &word, &mask); + *word &= ~mask; + } + + MOZ_ALWAYS_INLINE void copyMarkBit(Cell* dst, const TenuredCell* src, uint32_t color) { + uintptr_t* word, mask; + getMarkWordAndMask(dst, color, &word, &mask); + *word = (*word & ~mask) | (src->isMarked(color) ? mask : 0); + } + + void clear() { + memset((void*)bitmap, 0, sizeof(bitmap)); + } + + uintptr_t* arenaBits(Arena* arena) { + static_assert(ArenaBitmapBits == ArenaBitmapWords * JS_BITS_PER_WORD, + "We assume that the part of the bitmap corresponding to the arena " + "has the exact number of words so we do not need to deal with a word " + "that covers bits from two arenas."); + + uintptr_t* word, unused; + getMarkWordAndMask(reinterpret_cast<Cell*>(arena->address()), BLACK, &word, &unused); + return word; + } +}; + +static_assert(ArenaBitmapBytes * ArenasPerChunk == sizeof(ChunkBitmap), + "Ensure our ChunkBitmap actually covers all arenas."); +static_assert(js::gc::ChunkMarkBitmapBits == ArenaBitmapBits * ArenasPerChunk, + "Ensure that the mark bitmap has the right number of bits."); + +typedef BitArray<ArenasPerChunk> PerArenaBitmap; + +const size_t ChunkPadSize = ChunkSize + - (sizeof(Arena) * ArenasPerChunk) + - sizeof(ChunkBitmap) + - sizeof(PerArenaBitmap) + - sizeof(ChunkInfo) + - sizeof(ChunkTrailer); +static_assert(ChunkPadSize < BytesPerArenaWithHeader, + "If the chunk padding is larger than an arena, we should have one more arena."); + +/* + * Chunks contain arenas and associated data structures (mark bitmap, delayed + * marking state). + */ +struct Chunk +{ + Arena arenas[ArenasPerChunk]; + + /* Pad to full size to ensure cache alignment of ChunkInfo. */ + uint8_t padding[ChunkPadSize]; + + ChunkBitmap bitmap; + PerArenaBitmap decommittedArenas; + ChunkInfo info; + ChunkTrailer trailer; + + static Chunk* fromAddress(uintptr_t addr) { + addr &= ~ChunkMask; + return reinterpret_cast<Chunk*>(addr); + } + + static bool withinValidRange(uintptr_t addr) { + uintptr_t offset = addr & ChunkMask; + return Chunk::fromAddress(addr)->isNurseryChunk() + ? offset < ChunkSize - sizeof(ChunkTrailer) + : offset < ArenasPerChunk * ArenaSize; + } + + static size_t arenaIndex(uintptr_t addr) { + MOZ_ASSERT(!Chunk::fromAddress(addr)->isNurseryChunk()); + MOZ_ASSERT(withinValidRange(addr)); + return (addr & ChunkMask) >> ArenaShift; + } + + uintptr_t address() const { + uintptr_t addr = reinterpret_cast<uintptr_t>(this); + MOZ_ASSERT(!(addr & ChunkMask)); + return addr; + } + + bool unused() const { + return info.numArenasFree == ArenasPerChunk; + } + + bool hasAvailableArenas() const { + return info.numArenasFree != 0; + } + + bool isNurseryChunk() const { + return trailer.storeBuffer; + } + + Arena* allocateArena(JSRuntime* rt, JS::Zone* zone, AllocKind kind, const AutoLockGC& lock); + + void releaseArena(JSRuntime* rt, Arena* arena, const AutoLockGC& lock); + void recycleArena(Arena* arena, SortedArenaList& dest, size_t thingsPerArena); + + MOZ_MUST_USE bool decommitOneFreeArena(JSRuntime* rt, AutoLockGC& lock); + void decommitAllArenasWithoutUnlocking(const AutoLockGC& lock); + + static Chunk* allocate(JSRuntime* rt); + void init(JSRuntime* rt); + + private: + void decommitAllArenas(JSRuntime* rt); + + /* Search for a decommitted arena to allocate. */ + unsigned findDecommittedArenaOffset(); + Arena* fetchNextDecommittedArena(); + + void addArenaToFreeList(JSRuntime* rt, Arena* arena); + void addArenaToDecommittedList(JSRuntime* rt, const Arena* arena); + + void updateChunkListAfterAlloc(JSRuntime* rt, const AutoLockGC& lock); + void updateChunkListAfterFree(JSRuntime* rt, const AutoLockGC& lock); + + public: + /* Unlink and return the freeArenasHead. */ + Arena* fetchNextFreeArena(JSRuntime* rt); +}; + +static_assert(sizeof(Chunk) == ChunkSize, + "Ensure the hardcoded chunk size definition actually matches the struct."); +static_assert(js::gc::ChunkMarkBitmapOffset == offsetof(Chunk, bitmap), + "The hardcoded API bitmap offset must match the actual offset."); +static_assert(js::gc::ChunkRuntimeOffset == offsetof(Chunk, trailer) + + offsetof(ChunkTrailer, runtime), + "The hardcoded API runtime offset must match the actual offset."); +static_assert(js::gc::ChunkLocationOffset == offsetof(Chunk, trailer) + + offsetof(ChunkTrailer, location), + "The hardcoded API location offset must match the actual offset."); + +/* + * Tracks the used sizes for owned heap data and automatically maintains the + * memory usage relationship between GCRuntime and Zones. + */ +class HeapUsage +{ + /* + * A heap usage that contains our parent's heap usage, or null if this is + * the top-level usage container. + */ + HeapUsage* parent_; + + /* + * The approximate number of bytes in use on the GC heap, to the nearest + * ArenaSize. This does not include any malloc data. It also does not + * include not-actively-used addresses that are still reserved at the OS + * level for GC usage. It is atomic because it is updated by both the main + * and GC helper threads. + */ + mozilla::Atomic<size_t, mozilla::ReleaseAcquire> gcBytes_; + + public: + explicit HeapUsage(HeapUsage* parent) + : parent_(parent), + gcBytes_(0) + {} + + size_t gcBytes() const { return gcBytes_; } + + void addGCArena() { + gcBytes_ += ArenaSize; + if (parent_) + parent_->addGCArena(); + } + void removeGCArena() { + MOZ_ASSERT(gcBytes_ >= ArenaSize); + gcBytes_ -= ArenaSize; + if (parent_) + parent_->removeGCArena(); + } + + /* Pair to adoptArenas. Adopts the attendant usage statistics. */ + void adopt(HeapUsage& other) { + gcBytes_ += other.gcBytes_; + other.gcBytes_ = 0; + } +}; + +inline void +Arena::checkAddress() const +{ + mozilla::DebugOnly<uintptr_t> addr = uintptr_t(this); + MOZ_ASSERT(addr); + MOZ_ASSERT(!(addr & ArenaMask)); + MOZ_ASSERT(Chunk::withinValidRange(addr)); +} + +inline Chunk* +Arena::chunk() const +{ + return Chunk::fromAddress(address()); +} + +static void +AssertValidColor(const TenuredCell* thing, uint32_t color) +{ +#ifdef DEBUG + Arena* arena = thing->arena(); + MOZ_ASSERT(color < arena->getThingSize() / CellSize); +#endif +} + +MOZ_ALWAYS_INLINE const TenuredCell& +Cell::asTenured() const +{ + MOZ_ASSERT(isTenured()); + return *static_cast<const TenuredCell*>(this); +} + +MOZ_ALWAYS_INLINE TenuredCell& +Cell::asTenured() +{ + MOZ_ASSERT(isTenured()); + return *static_cast<TenuredCell*>(this); +} + +inline JSRuntime* +Cell::runtimeFromMainThread() const +{ + JSRuntime* rt = chunk()->trailer.runtime; + MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt)); + return rt; +} + +inline JS::shadow::Runtime* +Cell::shadowRuntimeFromMainThread() const +{ + return reinterpret_cast<JS::shadow::Runtime*>(runtimeFromMainThread()); +} + +inline JSRuntime* +Cell::runtimeFromAnyThread() const +{ + return chunk()->trailer.runtime; +} + +inline JS::shadow::Runtime* +Cell::shadowRuntimeFromAnyThread() const +{ + return reinterpret_cast<JS::shadow::Runtime*>(runtimeFromAnyThread()); +} + +inline uintptr_t +Cell::address() const +{ + uintptr_t addr = uintptr_t(this); + MOZ_ASSERT(addr % CellSize == 0); + MOZ_ASSERT(Chunk::withinValidRange(addr)); + return addr; +} + +Chunk* +Cell::chunk() const +{ + uintptr_t addr = uintptr_t(this); + MOZ_ASSERT(addr % CellSize == 0); + addr &= ~ChunkMask; + return reinterpret_cast<Chunk*>(addr); +} + +inline StoreBuffer* +Cell::storeBuffer() const +{ + return chunk()->trailer.storeBuffer; +} + +inline JS::TraceKind +Cell::getTraceKind() const +{ + return isTenured() ? asTenured().getTraceKind() : JS::TraceKind::Object; +} + +inline bool +InFreeList(Arena* arena, void* thing) +{ + uintptr_t addr = reinterpret_cast<uintptr_t>(thing); + MOZ_ASSERT(Arena::isAligned(addr, arena->getThingSize())); + return arena->inFreeList(addr); +} + +/* static */ MOZ_ALWAYS_INLINE bool +Cell::needWriteBarrierPre(JS::Zone* zone) { + return JS::shadow::Zone::asShadowZone(zone)->needsIncrementalBarrier(); +} + +/* static */ MOZ_ALWAYS_INLINE TenuredCell* +TenuredCell::fromPointer(void* ptr) +{ + MOZ_ASSERT(static_cast<TenuredCell*>(ptr)->isTenured()); + return static_cast<TenuredCell*>(ptr); +} + +/* static */ MOZ_ALWAYS_INLINE const TenuredCell* +TenuredCell::fromPointer(const void* ptr) +{ + MOZ_ASSERT(static_cast<const TenuredCell*>(ptr)->isTenured()); + return static_cast<const TenuredCell*>(ptr); +} + +bool +TenuredCell::isMarked(uint32_t color /* = BLACK */) const +{ + MOZ_ASSERT(arena()->allocated()); + AssertValidColor(this, color); + return chunk()->bitmap.isMarked(this, color); +} + +bool +TenuredCell::markIfUnmarked(uint32_t color /* = BLACK */) const +{ + AssertValidColor(this, color); + return chunk()->bitmap.markIfUnmarked(this, color); +} + +void +TenuredCell::unmark(uint32_t color) const +{ + MOZ_ASSERT(color != BLACK); + AssertValidColor(this, color); + chunk()->bitmap.unmark(this, color); +} + +void +TenuredCell::copyMarkBitsFrom(const TenuredCell* src) +{ + ChunkBitmap& bitmap = chunk()->bitmap; + bitmap.copyMarkBit(this, src, BLACK); + bitmap.copyMarkBit(this, src, GRAY); +} + +inline Arena* +TenuredCell::arena() const +{ + MOZ_ASSERT(isTenured()); + uintptr_t addr = address(); + addr &= ~ArenaMask; + return reinterpret_cast<Arena*>(addr); +} + +AllocKind +TenuredCell::getAllocKind() const +{ + return arena()->getAllocKind(); +} + +JS::TraceKind +TenuredCell::getTraceKind() const +{ + return MapAllocToTraceKind(getAllocKind()); +} + +JS::Zone* +TenuredCell::zone() const +{ + JS::Zone* zone = arena()->zone; + MOZ_ASSERT(CurrentThreadCanAccessZone(zone)); + return zone; +} + +JS::Zone* +TenuredCell::zoneFromAnyThread() const +{ + return arena()->zone; +} + +bool +TenuredCell::isInsideZone(JS::Zone* zone) const +{ + return zone == arena()->zone; +} + +/* static */ MOZ_ALWAYS_INLINE void +TenuredCell::readBarrier(TenuredCell* thing) +{ + MOZ_ASSERT(!CurrentThreadIsIonCompiling()); + MOZ_ASSERT(!isNullLike(thing)); + + // It would be good if barriers were never triggered during collection, but + // at the moment this can happen e.g. when rekeying tables containing + // read-barriered GC things after a moving GC. + // + // TODO: Fix this and assert we're not collecting if we're on the main + // thread. + + JS::shadow::Zone* shadowZone = thing->shadowZoneFromAnyThread(); + if (shadowZone->needsIncrementalBarrier()) { + // Barriers are only enabled on the main thread and are disabled while collecting. + MOZ_ASSERT(!RuntimeFromMainThreadIsHeapMajorCollecting(shadowZone)); + Cell* tmp = thing; + TraceManuallyBarrieredGenericPointerEdge(shadowZone->barrierTracer(), &tmp, "read barrier"); + MOZ_ASSERT(tmp == thing); + } + + if (thing->isMarked(GRAY)) { + // There shouldn't be anything marked grey unless we're on the main thread. + MOZ_ASSERT(CurrentThreadCanAccessRuntime(thing->runtimeFromAnyThread())); + if (!RuntimeFromMainThreadIsHeapMajorCollecting(shadowZone)) + UnmarkGrayCellRecursively(thing, thing->getTraceKind()); + } +} + +void +AssertSafeToSkipBarrier(TenuredCell* thing); + +/* static */ MOZ_ALWAYS_INLINE void +TenuredCell::writeBarrierPre(TenuredCell* thing) +{ + MOZ_ASSERT(!CurrentThreadIsIonCompiling()); + MOZ_ASSERT_IF(thing, !isNullLike(thing)); + if (!thing) + return; + +#ifdef JS_GC_ZEAL + // When verifying pre barriers we need to switch on all barriers, even + // those on the Atoms Zone. Normally, we never enter a parse task when + // collecting in the atoms zone, so will filter out atoms below. + // Unfortuantely, If we try that when verifying pre-barriers, we'd never be + // able to handle OMT parse tasks at all as we switch on the verifier any + // time we're not doing GC. This would cause us to deadlock, as OMT parsing + // is meant to resume after GC work completes. Instead we filter out any + // OMT barriers that reach us and assert that they would normally not be + // possible. + if (!CurrentThreadCanAccessRuntime(thing->runtimeFromAnyThread())) { + AssertSafeToSkipBarrier(thing); + return; + } +#endif + + JS::shadow::Zone* shadowZone = thing->shadowZoneFromAnyThread(); + if (shadowZone->needsIncrementalBarrier()) { + MOZ_ASSERT(!RuntimeFromMainThreadIsHeapMajorCollecting(shadowZone)); + Cell* tmp = thing; + TraceManuallyBarrieredGenericPointerEdge(shadowZone->barrierTracer(), &tmp, "pre barrier"); + MOZ_ASSERT(tmp == thing); + } +} + +static MOZ_ALWAYS_INLINE void +AssertValidToSkipBarrier(TenuredCell* thing) +{ + MOZ_ASSERT(!IsInsideNursery(thing)); + MOZ_ASSERT_IF(thing, MapAllocToTraceKind(thing->getAllocKind()) != JS::TraceKind::Object); +} + +/* static */ MOZ_ALWAYS_INLINE void +TenuredCell::writeBarrierPost(void* cellp, TenuredCell* prior, TenuredCell* next) +{ + AssertValidToSkipBarrier(next); +} + +#ifdef DEBUG +bool +Cell::isAligned() const +{ + if (!isTenured()) + return true; + return asTenured().isAligned(); +} + +bool +TenuredCell::isAligned() const +{ + return Arena::isAligned(address(), arena()->getThingSize()); +} +#endif + +static const int32_t ChunkLocationOffsetFromLastByte = + int32_t(gc::ChunkLocationOffset) - int32_t(gc::ChunkMask); + +} /* namespace gc */ +} /* namespace js */ + +#endif /* gc_Heap_h */ diff --git a/js/src/gc/Iteration.cpp b/js/src/gc/Iteration.cpp new file mode 100644 index 000000000..3ebd5d290 --- /dev/null +++ b/js/src/gc/Iteration.cpp @@ -0,0 +1,142 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/DebugOnly.h" + +#include "jscompartment.h" +#include "jsgc.h" + +#include "gc/GCInternals.h" +#include "js/HashTable.h" +#include "vm/Runtime.h" + +#include "jscntxtinlines.h" +#include "jsgcinlines.h" + +using namespace js; +using namespace js::gc; + +static void +IterateCompartmentsArenasCells(JSContext* cx, Zone* zone, void* data, + JSIterateCompartmentCallback compartmentCallback, + IterateArenaCallback arenaCallback, + IterateCellCallback cellCallback) +{ + for (CompartmentsInZoneIter comp(zone); !comp.done(); comp.next()) + (*compartmentCallback)(cx, data, comp); + + for (auto thingKind : AllAllocKinds()) { + JS::TraceKind traceKind = MapAllocToTraceKind(thingKind); + size_t thingSize = Arena::thingSize(thingKind); + + for (ArenaIter aiter(zone, thingKind); !aiter.done(); aiter.next()) { + Arena* arena = aiter.get(); + (*arenaCallback)(cx, data, arena, traceKind, thingSize); + for (ArenaCellIter iter(arena); !iter.done(); iter.next()) + (*cellCallback)(cx, data, iter.getCell(), traceKind, thingSize); + } + } +} + +void +js::IterateZonesCompartmentsArenasCells(JSContext* cx, void* data, + IterateZoneCallback zoneCallback, + JSIterateCompartmentCallback compartmentCallback, + IterateArenaCallback arenaCallback, + IterateCellCallback cellCallback) +{ + AutoPrepareForTracing prop(cx, WithAtoms); + + for (ZonesIter zone(cx, WithAtoms); !zone.done(); zone.next()) { + (*zoneCallback)(cx, data, zone); + IterateCompartmentsArenasCells(cx, zone, data, + compartmentCallback, arenaCallback, cellCallback); + } +} + +void +js::IterateZoneCompartmentsArenasCells(JSContext* cx, Zone* zone, void* data, + IterateZoneCallback zoneCallback, + JSIterateCompartmentCallback compartmentCallback, + IterateArenaCallback arenaCallback, + IterateCellCallback cellCallback) +{ + AutoPrepareForTracing prop(cx, WithAtoms); + + (*zoneCallback)(cx, data, zone); + IterateCompartmentsArenasCells(cx, zone, data, + compartmentCallback, arenaCallback, cellCallback); +} + +void +js::IterateChunks(JSContext* cx, void* data, IterateChunkCallback chunkCallback) +{ + AutoPrepareForTracing prep(cx, SkipAtoms); + + for (auto chunk = cx->gc.allNonEmptyChunks(); !chunk.done(); chunk.next()) + chunkCallback(cx, data, chunk); +} + +void +js::IterateScripts(JSContext* cx, JSCompartment* compartment, + void* data, IterateScriptCallback scriptCallback) +{ + MOZ_ASSERT(!cx->mainThread().suppressGC); + AutoEmptyNursery empty(cx); + AutoPrepareForTracing prep(cx, SkipAtoms); + + if (compartment) { + Zone* zone = compartment->zone(); + for (auto script = zone->cellIter<JSScript>(empty); !script.done(); script.next()) { + if (script->compartment() == compartment) + scriptCallback(cx, data, script); + } + } else { + for (ZonesIter zone(cx, SkipAtoms); !zone.done(); zone.next()) { + for (auto script = zone->cellIter<JSScript>(empty); !script.done(); script.next()) + scriptCallback(cx, data, script); + } + } +} + +static void +IterateGrayObjects(Zone* zone, GCThingCallback cellCallback, void* data) +{ + for (auto kind : ObjectAllocKinds()) { + for (GrayObjectIter obj(zone, kind); !obj.done(); obj.next()) { + if (obj->asTenured().isMarked(GRAY)) + cellCallback(data, JS::GCCellPtr(obj.get())); + } + } +} + +void +js::IterateGrayObjects(Zone* zone, GCThingCallback cellCallback, void* data) +{ + JSRuntime* rt = zone->runtimeFromMainThread(); + MOZ_ASSERT(!rt->isHeapBusy()); + AutoPrepareForTracing prep(rt->contextFromMainThread(), SkipAtoms); + ::IterateGrayObjects(zone, cellCallback, data); +} + +void +js::IterateGrayObjectsUnderCC(Zone* zone, GCThingCallback cellCallback, void* data) +{ + mozilla::DebugOnly<JSRuntime*> rt = zone->runtimeFromMainThread(); + MOZ_ASSERT(rt->isCycleCollecting()); + MOZ_ASSERT(!rt->gc.isIncrementalGCInProgress()); + ::IterateGrayObjects(zone, cellCallback, data); +} + +JS_PUBLIC_API(void) +JS_IterateCompartments(JSContext* cx, void* data, + JSIterateCompartmentCallback compartmentCallback) +{ + AutoTraceSession session(cx); + + for (CompartmentsIter c(cx, WithAtoms); !c.done(); c.next()) + (*compartmentCallback)(cx, data, c); +} diff --git a/js/src/gc/Marking.cpp b/js/src/gc/Marking.cpp new file mode 100644 index 000000000..d9235f9ac --- /dev/null +++ b/js/src/gc/Marking.cpp @@ -0,0 +1,3019 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/Marking.h" + +#include "mozilla/DebugOnly.h" +#include "mozilla/IntegerRange.h" +#include "mozilla/ReentrancyGuard.h" +#include "mozilla/ScopeExit.h" +#include "mozilla/TypeTraits.h" + +#include "jsgc.h" +#include "jsprf.h" + +#include "builtin/ModuleObject.h" +#include "gc/GCInternals.h" +#include "gc/Policy.h" +#include "jit/IonCode.h" +#include "js/SliceBudget.h" +#include "vm/ArgumentsObject.h" +#include "vm/ArrayObject.h" +#include "vm/Debugger.h" +#include "vm/EnvironmentObject.h" +#include "vm/Scope.h" +#include "vm/Shape.h" +#include "vm/Symbol.h" +#include "vm/TypedArrayObject.h" +#include "vm/UnboxedObject.h" +#include "wasm/WasmJS.h" + +#include "jscompartmentinlines.h" +#include "jsgcinlines.h" +#include "jsobjinlines.h" + +#include "gc/Nursery-inl.h" +#include "vm/String-inl.h" +#include "vm/UnboxedObject-inl.h" + +using namespace js; +using namespace js::gc; + +using JS::MapTypeToTraceKind; + +using mozilla::ArrayLength; +using mozilla::DebugOnly; +using mozilla::IsBaseOf; +using mozilla::IsSame; +using mozilla::MakeRange; +using mozilla::PodCopy; + +// Tracing Overview +// ================ +// +// Tracing, in this context, refers to an abstract visitation of some or all of +// the GC-controlled heap. The effect of tracing an edge of the graph depends +// on the subclass of the JSTracer on whose behalf we are tracing. +// +// Marking +// ------- +// +// The primary JSTracer is the GCMarker. The marking tracer causes the target +// of each traversed edge to be marked black and the target edge's children to +// be marked either gray (in the gc algorithm sense) or immediately black. +// +// Callback +// -------- +// +// The secondary JSTracer is the CallbackTracer. This simply invokes a callback +// on each edge in a child. +// +// The following is a rough outline of the general struture of the tracing +// internals. +// +// // +// .---------. .---------. .--------------------------. .----------. // +// |TraceEdge| |TraceRoot| |TraceManuallyBarrieredEdge| ... |TraceRange| ... etc. // +// '---------' '---------' '--------------------------' '----------' // +// \ \ / / // +// \ \ .----------------. / / // +// o------------->o-|DispatchToTracer|-o<-----------------------o // +// '----------------' // +// / \ // +// / \ // +// .---------. .----------. .-----------------. // +// |DoMarking| |DoCallback|-------> |<JSTraceCallback>|-----------> // +// '---------' '----------' '-----------------' // +// | // +// | // +// .--------. // +// o---------------->|traverse| . // +// /_\ '--------' ' . // +// | . . ' . // +// | . . ' . // +// | . . ' . // +// | .-----------. .-----------. ' . .--------------------. // +// | |markAndScan| |markAndPush| ' - |markAndTraceChildren|----> // +// | '-----------' '-----------' '--------------------' // +// | | \ // +// | | \ // +// | .----------------------. .----------------. // +// | |T::eagerlyMarkChildren| |pushMarkStackTop|<===Oo // +// | '----------------------' '----------------' || // +// | | || || // +// | | || || // +// | | || || // +// o<-----------------o<========================OO============Oo // +// // +// // +// Legend: // +// ------ Direct calls // +// . . . Static dispatch // +// ====== Dispatch through a manual stack. // +// // + + +/*** Tracing Invariants **************************************************************************/ + +#if defined(DEBUG) +template<typename T> +static inline bool +IsThingPoisoned(T* thing) +{ + const uint8_t poisonBytes[] = { + JS_FRESH_NURSERY_PATTERN, + JS_SWEPT_NURSERY_PATTERN, + JS_ALLOCATED_NURSERY_PATTERN, + JS_FRESH_TENURED_PATTERN, + JS_MOVED_TENURED_PATTERN, + JS_SWEPT_TENURED_PATTERN, + JS_ALLOCATED_TENURED_PATTERN, + JS_SWEPT_CODE_PATTERN + }; + const int numPoisonBytes = sizeof(poisonBytes) / sizeof(poisonBytes[0]); + uint32_t* p = reinterpret_cast<uint32_t*>(reinterpret_cast<FreeSpan*>(thing) + 1); + // Note: all free patterns are odd to make the common, not-poisoned case a single test. + if ((*p & 1) == 0) + return false; + for (int i = 0; i < numPoisonBytes; ++i) { + const uint8_t pb = poisonBytes[i]; + const uint32_t pw = pb | (pb << 8) | (pb << 16) | (pb << 24); + if (*p == pw) + return true; + } + return false; +} + +static bool +IsMovingTracer(JSTracer *trc) +{ + return trc->isCallbackTracer() && + trc->asCallbackTracer()->getTracerKind() == JS::CallbackTracer::TracerKind::Moving; +} +#endif + +template <typename T> bool ThingIsPermanentAtomOrWellKnownSymbol(T* thing) { return false; } +template <> bool ThingIsPermanentAtomOrWellKnownSymbol<JSString>(JSString* str) { + return str->isPermanentAtom(); +} +template <> bool ThingIsPermanentAtomOrWellKnownSymbol<JSFlatString>(JSFlatString* str) { + return str->isPermanentAtom(); +} +template <> bool ThingIsPermanentAtomOrWellKnownSymbol<JSLinearString>(JSLinearString* str) { + return str->isPermanentAtom(); +} +template <> bool ThingIsPermanentAtomOrWellKnownSymbol<JSAtom>(JSAtom* atom) { + return atom->isPermanent(); +} +template <> bool ThingIsPermanentAtomOrWellKnownSymbol<PropertyName>(PropertyName* name) { + return name->isPermanent(); +} +template <> bool ThingIsPermanentAtomOrWellKnownSymbol<JS::Symbol>(JS::Symbol* sym) { + return sym->isWellKnownSymbol(); +} + +template <typename T> +static inline bool +IsOwnedByOtherRuntime(JSRuntime* rt, T thing) +{ + bool other = thing->runtimeFromAnyThread() != rt; + MOZ_ASSERT_IF(other, + ThingIsPermanentAtomOrWellKnownSymbol(thing) || + thing->zoneFromAnyThread()->isSelfHostingZone()); + return other; +} + +template<typename T> +void +js::CheckTracedThing(JSTracer* trc, T* thing) +{ +#ifdef DEBUG + MOZ_ASSERT(trc); + MOZ_ASSERT(thing); + + if (!trc->checkEdges()) + return; + + if (IsForwarded(thing)) + thing = Forwarded(thing); + + /* This function uses data that's not available in the nursery. */ + if (IsInsideNursery(thing)) + return; + + MOZ_ASSERT_IF(!IsMovingTracer(trc) && !trc->isTenuringTracer(), !IsForwarded(thing)); + + /* + * Permanent atoms and things in the self-hosting zone are not associated + * with this runtime, but will be ignored during marking. + */ + if (IsOwnedByOtherRuntime(trc->runtime(), thing)) + return; + + Zone* zone = thing->zoneFromAnyThread(); + JSRuntime* rt = trc->runtime(); + + MOZ_ASSERT_IF(!IsMovingTracer(trc), CurrentThreadCanAccessZone(zone)); + MOZ_ASSERT_IF(!IsMovingTracer(trc), CurrentThreadCanAccessRuntime(rt)); + + MOZ_ASSERT(zone->runtimeFromAnyThread() == trc->runtime()); + + MOZ_ASSERT(thing->isAligned()); + MOZ_ASSERT(MapTypeToTraceKind<typename mozilla::RemovePointer<T>::Type>::kind == + thing->getTraceKind()); + + /* + * Do not check IsMarkingTracer directly -- it should only be used in paths + * where we cannot be the gray buffering tracer. + */ + bool isGcMarkingTracer = trc->isMarkingTracer(); + + MOZ_ASSERT_IF(zone->requireGCTracer(), isGcMarkingTracer || IsBufferGrayRootsTracer(trc)); + + if (isGcMarkingTracer) { + GCMarker* gcMarker = static_cast<GCMarker*>(trc); + MOZ_ASSERT_IF(gcMarker->shouldCheckCompartments(), + zone->isCollecting() || zone->isAtomsZone()); + + MOZ_ASSERT_IF(gcMarker->markColor() == GRAY, + !zone->isGCMarkingBlack() || zone->isAtomsZone()); + + MOZ_ASSERT(!(zone->isGCSweeping() || zone->isGCFinished() || zone->isGCCompacting())); + } + + /* + * Try to assert that the thing is allocated. + * + * We would like to assert that the thing is not in the free list, but this + * check is very slow. Instead we check whether the thing has been poisoned: + * if it has not then we assume it is allocated, but if it has then it is + * either free or uninitialized in which case we check the free list. + * + * Further complications are that background sweeping may be running and + * concurrently modifiying the free list and that tracing is done off main + * thread during compacting GC and reading the contents of the thing by + * IsThingPoisoned would be racy in this case. + */ + MOZ_ASSERT_IF(rt->isHeapBusy() && !zone->isGCCompacting() && !rt->gc.isBackgroundSweeping(), + !IsThingPoisoned(thing) || !InFreeList(thing->asTenured().arena(), thing)); +#endif +} + +template <typename S> +struct CheckTracedFunctor : public VoidDefaultAdaptor<S> { + template <typename T> void operator()(T* t, JSTracer* trc) { CheckTracedThing(trc, t); } +}; + +template<typename T> +void +js::CheckTracedThing(JSTracer* trc, T thing) +{ + DispatchTyped(CheckTracedFunctor<T>(), thing, trc); +} + +namespace js { +#define IMPL_CHECK_TRACED_THING(_, type, __) \ + template void CheckTracedThing<type>(JSTracer*, type*); +JS_FOR_EACH_TRACEKIND(IMPL_CHECK_TRACED_THING); +#undef IMPL_CHECK_TRACED_THING +} // namespace js + +static bool +ShouldMarkCrossCompartment(JSTracer* trc, JSObject* src, Cell* cell) +{ + if (!trc->isMarkingTracer()) + return true; + + uint32_t color = static_cast<GCMarker*>(trc)->markColor(); + MOZ_ASSERT(color == BLACK || color == GRAY); + + if (!cell->isTenured()) { + MOZ_ASSERT(color == BLACK); + return false; + } + TenuredCell& tenured = cell->asTenured(); + + JS::Zone* zone = tenured.zone(); + if (color == BLACK) { + /* + * Having black->gray edges violates our promise to the cycle + * collector. This can happen if we're collecting a compartment and it + * has an edge to an uncollected compartment: it's possible that the + * source and destination of the cross-compartment edge should be gray, + * but the source was marked black by the conservative scanner. + */ + if (tenured.isMarked(GRAY)) { + MOZ_ASSERT(!zone->isCollecting()); + trc->runtime()->gc.setFoundBlackGrayEdges(tenured); + } + return zone->isGCMarking(); + } else { + if (zone->isGCMarkingBlack()) { + /* + * The destination compartment is being not being marked gray now, + * but it will be later, so record the cell so it can be marked gray + * at the appropriate time. + */ + if (!tenured.isMarked()) + DelayCrossCompartmentGrayMarking(src); + return false; + } + return zone->isGCMarkingGray(); + } +} + +static bool +ShouldMarkCrossCompartment(JSTracer* trc, JSObject* src, const Value& val) +{ + return val.isMarkable() && ShouldMarkCrossCompartment(trc, src, (Cell*)val.toGCThing()); +} + +static void +AssertZoneIsMarking(Cell* thing) +{ + MOZ_ASSERT(TenuredCell::fromPointer(thing)->zone()->isGCMarking()); +} + +static void +AssertZoneIsMarking(JSString* str) +{ +#ifdef DEBUG + Zone* zone = TenuredCell::fromPointer(str)->zone(); + MOZ_ASSERT(zone->isGCMarking() || zone->isAtomsZone()); +#endif +} + +static void +AssertZoneIsMarking(JS::Symbol* sym) +{ +#ifdef DEBUG + Zone* zone = TenuredCell::fromPointer(sym)->zone(); + MOZ_ASSERT(zone->isGCMarking() || zone->isAtomsZone()); +#endif +} + +static void +AssertRootMarkingPhase(JSTracer* trc) +{ + MOZ_ASSERT_IF(trc->isMarkingTracer(), + trc->runtime()->gc.state() == State::NotActive || + trc->runtime()->gc.state() == State::MarkRoots); +} + + +/*** Tracing Interface ***************************************************************************/ + +// The second parameter to BaseGCType is derived automatically based on T. The +// relation here is that for any T, the TraceKind will automatically, +// statically select the correct Cell layout for marking. Below, we instantiate +// each override with a declaration of the most derived layout type. +// +// The use of TraceKind::Null for the case where the type is not matched +// generates a compile error as no template instantiated for that kind. +// +// Usage: +// BaseGCType<T>::type +// +// Examples: +// BaseGCType<JSFunction>::type => JSObject +// BaseGCType<UnownedBaseShape>::type => BaseShape +// etc. +template <typename T, JS::TraceKind = +#define EXPAND_MATCH_TYPE(name, type, _) \ + IsBaseOf<type, T>::value ? JS::TraceKind::name : +JS_FOR_EACH_TRACEKIND(EXPAND_MATCH_TYPE) +#undef EXPAND_MATCH_TYPE + JS::TraceKind::Null> + +struct BaseGCType; +#define IMPL_BASE_GC_TYPE(name, type_, _) \ + template <typename T> struct BaseGCType<T, JS::TraceKind:: name> { typedef type_ type; }; +JS_FOR_EACH_TRACEKIND(IMPL_BASE_GC_TYPE); +#undef IMPL_BASE_GC_TYPE + +// Our barrier templates are parameterized on the pointer types so that we can +// share the definitions with Value and jsid. Thus, we need to strip the +// pointer before sending the type to BaseGCType and re-add it on the other +// side. As such: +template <typename T> struct PtrBaseGCType { typedef T type; }; +template <typename T> struct PtrBaseGCType<T*> { typedef typename BaseGCType<T>::type* type; }; + +template <typename T> +typename PtrBaseGCType<T>::type* +ConvertToBase(T* thingp) +{ + return reinterpret_cast<typename PtrBaseGCType<T>::type*>(thingp); +} + +template <typename T> void DispatchToTracer(JSTracer* trc, T* thingp, const char* name); +template <typename T> T DoCallback(JS::CallbackTracer* trc, T* thingp, const char* name); +template <typename T> void DoMarking(GCMarker* gcmarker, T* thing); +template <typename T> void DoMarking(GCMarker* gcmarker, const T& thing); +template <typename T> void NoteWeakEdge(GCMarker* gcmarker, T** thingp); +template <typename T> void NoteWeakEdge(GCMarker* gcmarker, T* thingp); + +template <typename T> +void +js::TraceEdge(JSTracer* trc, WriteBarrieredBase<T>* thingp, const char* name) +{ + DispatchToTracer(trc, ConvertToBase(thingp->unsafeUnbarrieredForTracing()), name); +} + +template <typename T> +void +js::TraceEdge(JSTracer* trc, ReadBarriered<T>* thingp, const char* name) +{ + DispatchToTracer(trc, ConvertToBase(thingp->unsafeGet()), name); +} + +template <typename T> +void +js::TraceNullableEdge(JSTracer* trc, WriteBarrieredBase<T>* thingp, const char* name) +{ + if (InternalBarrierMethods<T>::isMarkable(thingp->get())) + DispatchToTracer(trc, ConvertToBase(thingp->unsafeUnbarrieredForTracing()), name); +} + +template <typename T> +JS_PUBLIC_API(void) +JS::TraceEdge(JSTracer* trc, JS::Heap<T>* thingp, const char* name) +{ + MOZ_ASSERT(thingp); + if (InternalBarrierMethods<T>::isMarkable(*thingp->unsafeGet())) + DispatchToTracer(trc, ConvertToBase(thingp->unsafeGet()), name); +} + +JS_PUBLIC_API(void) +JS::TraceEdge(JSTracer* trc, JS::TenuredHeap<JSObject*>* thingp, const char* name) +{ + MOZ_ASSERT(thingp); + if (JSObject* ptr = thingp->unbarrieredGetPtr()) { + DispatchToTracer(trc, &ptr, name); + thingp->setPtr(ptr); + } +} + +template <typename T> +void +js::TraceManuallyBarrieredEdge(JSTracer* trc, T* thingp, const char* name) +{ + DispatchToTracer(trc, ConvertToBase(thingp), name); +} + +template <typename T> +JS_PUBLIC_API(void) +js::UnsafeTraceManuallyBarrieredEdge(JSTracer* trc, T* thingp, const char* name) +{ + DispatchToTracer(trc, ConvertToBase(thingp), name); +} + +template <typename T> +void +js::TraceWeakEdge(JSTracer* trc, WeakRef<T>* thingp, const char* name) +{ + // Non-marking tracers treat the edge strongly. + if (!trc->isMarkingTracer()) + return DispatchToTracer(trc, ConvertToBase(thingp->unsafeUnbarrieredForTracing()), name); + + NoteWeakEdge(static_cast<GCMarker*>(trc), + ConvertToBase(thingp->unsafeUnbarrieredForTracing())); +} + +template <typename T> +void +js::TraceRoot(JSTracer* trc, T* thingp, const char* name) +{ + AssertRootMarkingPhase(trc); + DispatchToTracer(trc, ConvertToBase(thingp), name); +} + +template <typename T> +void +js::TraceRoot(JSTracer* trc, ReadBarriered<T>* thingp, const char* name) +{ + TraceRoot(trc, thingp->unsafeGet(), name); +} + +template <typename T> +void +js::TraceNullableRoot(JSTracer* trc, T* thingp, const char* name) +{ + AssertRootMarkingPhase(trc); + if (InternalBarrierMethods<T>::isMarkableTaggedPointer(*thingp)) + DispatchToTracer(trc, ConvertToBase(thingp), name); +} + +template <typename T> +void +js::TraceNullableRoot(JSTracer* trc, ReadBarriered<T>* thingp, const char* name) +{ + TraceNullableRoot(trc, thingp->unsafeGet(), name); +} + +template <typename T> +JS_PUBLIC_API(void) +JS::UnsafeTraceRoot(JSTracer* trc, T* thingp, const char* name) +{ + MOZ_ASSERT(thingp); + js::TraceNullableRoot(trc, thingp, name); +} + +template <typename T> +void +js::TraceRange(JSTracer* trc, size_t len, WriteBarrieredBase<T>* vec, const char* name) +{ + JS::AutoTracingIndex index(trc); + for (auto i : MakeRange(len)) { + if (InternalBarrierMethods<T>::isMarkable(vec[i].get())) + DispatchToTracer(trc, ConvertToBase(vec[i].unsafeUnbarrieredForTracing()), name); + ++index; + } +} + +template <typename T> +void +js::TraceRootRange(JSTracer* trc, size_t len, T* vec, const char* name) +{ + AssertRootMarkingPhase(trc); + JS::AutoTracingIndex index(trc); + for (auto i : MakeRange(len)) { + if (InternalBarrierMethods<T>::isMarkable(vec[i])) + DispatchToTracer(trc, ConvertToBase(&vec[i]), name); + ++index; + } +} + +// Instantiate a copy of the Tracing templates for each derived type. +#define INSTANTIATE_ALL_VALID_TRACE_FUNCTIONS(type) \ + template void js::TraceEdge<type>(JSTracer*, WriteBarrieredBase<type>*, const char*); \ + template void js::TraceEdge<type>(JSTracer*, ReadBarriered<type>*, const char*); \ + template void js::TraceNullableEdge<type>(JSTracer*, WriteBarrieredBase<type>*, const char*); \ + template void js::TraceManuallyBarrieredEdge<type>(JSTracer*, type*, const char*); \ + template void js::TraceWeakEdge<type>(JSTracer*, WeakRef<type>*, const char*); \ + template void js::TraceRoot<type>(JSTracer*, type*, const char*); \ + template void js::TraceRoot<type>(JSTracer*, ReadBarriered<type>*, const char*); \ + template void js::TraceNullableRoot<type>(JSTracer*, type*, const char*); \ + template void js::TraceNullableRoot<type>(JSTracer*, ReadBarriered<type>*, const char*); \ + template void js::TraceRange<type>(JSTracer*, size_t, WriteBarrieredBase<type>*, const char*); \ + template void js::TraceRootRange<type>(JSTracer*, size_t, type*, const char*); +FOR_EACH_GC_POINTER_TYPE(INSTANTIATE_ALL_VALID_TRACE_FUNCTIONS) +#undef INSTANTIATE_ALL_VALID_TRACE_FUNCTIONS + +#define INSTANTIATE_PUBLIC_TRACE_FUNCTIONS(type) \ + template JS_PUBLIC_API(void) JS::TraceEdge<type>(JSTracer*, JS::Heap<type>*, const char*); \ + template JS_PUBLIC_API(void) JS::UnsafeTraceRoot<type>(JSTracer*, type*, const char*); \ + template JS_PUBLIC_API(void) js::UnsafeTraceManuallyBarrieredEdge<type>(JSTracer*, type*, \ + const char*); +FOR_EACH_PUBLIC_GC_POINTER_TYPE(INSTANTIATE_PUBLIC_TRACE_FUNCTIONS) +FOR_EACH_PUBLIC_TAGGED_GC_POINTER_TYPE(INSTANTIATE_PUBLIC_TRACE_FUNCTIONS) +#undef INSTANTIATE_PUBLIC_TRACE_FUNCTIONS + +template <typename T> +void +js::TraceManuallyBarrieredCrossCompartmentEdge(JSTracer* trc, JSObject* src, T* dst, + const char* name) +{ + if (ShouldMarkCrossCompartment(trc, src, *dst)) + DispatchToTracer(trc, dst, name); +} +template void js::TraceManuallyBarrieredCrossCompartmentEdge<JSObject*>(JSTracer*, JSObject*, + JSObject**, const char*); +template void js::TraceManuallyBarrieredCrossCompartmentEdge<JSScript*>(JSTracer*, JSObject*, + JSScript**, const char*); + +template <typename T> +void +js::TraceCrossCompartmentEdge(JSTracer* trc, JSObject* src, WriteBarrieredBase<T>* dst, + const char* name) +{ + if (ShouldMarkCrossCompartment(trc, src, dst->get())) + DispatchToTracer(trc, dst->unsafeUnbarrieredForTracing(), name); +} +template void js::TraceCrossCompartmentEdge<Value>(JSTracer*, JSObject*, + WriteBarrieredBase<Value>*, const char*); + +template <typename T> +void +js::TraceProcessGlobalRoot(JSTracer* trc, T* thing, const char* name) +{ + AssertRootMarkingPhase(trc); + MOZ_ASSERT(ThingIsPermanentAtomOrWellKnownSymbol(thing)); + + // We have to mark permanent atoms and well-known symbols through a special + // method because the default DoMarking implementation automatically skips + // them. Fortunately, atoms (permanent and non) cannot refer to other GC + // things so they do not need to go through the mark stack and may simply + // be marked directly. Moreover, well-known symbols can refer only to + // permanent atoms, so likewise require no subsquent marking. + CheckTracedThing(trc, *ConvertToBase(&thing)); + if (trc->isMarkingTracer()) + thing->markIfUnmarked(gc::BLACK); + else + DoCallback(trc->asCallbackTracer(), ConvertToBase(&thing), name); +} +template void js::TraceProcessGlobalRoot<JSAtom>(JSTracer*, JSAtom*, const char*); +template void js::TraceProcessGlobalRoot<JS::Symbol>(JSTracer*, JS::Symbol*, const char*); + +// A typed functor adaptor for TraceRoot. +struct TraceRootFunctor { + template <typename T> + void operator()(JSTracer* trc, Cell** thingp, const char* name) { + TraceRoot(trc, reinterpret_cast<T**>(thingp), name); + } +}; + +void +js::TraceGenericPointerRoot(JSTracer* trc, Cell** thingp, const char* name) +{ + MOZ_ASSERT(thingp); + if (!*thingp) + return; + TraceRootFunctor f; + DispatchTraceKindTyped(f, (*thingp)->getTraceKind(), trc, thingp, name); +} + +// A typed functor adaptor for TraceManuallyBarrieredEdge. +struct TraceManuallyBarrieredEdgeFunctor { + template <typename T> + void operator()(JSTracer* trc, Cell** thingp, const char* name) { + TraceManuallyBarrieredEdge(trc, reinterpret_cast<T**>(thingp), name); + } +}; + +void +js::TraceManuallyBarrieredGenericPointerEdge(JSTracer* trc, Cell** thingp, const char* name) +{ + MOZ_ASSERT(thingp); + if (!*thingp) + return; + TraceManuallyBarrieredEdgeFunctor f; + DispatchTraceKindTyped(f, (*thingp)->getTraceKind(), trc, thingp, name); +} + +// This method is responsible for dynamic dispatch to the real tracer +// implementation. Consider replacing this choke point with virtual dispatch: +// a sufficiently smart C++ compiler may be able to devirtualize some paths. +template <typename T> +void +DispatchToTracer(JSTracer* trc, T* thingp, const char* name) +{ +#define IS_SAME_TYPE_OR(name, type, _) mozilla::IsSame<type*, T>::value || + static_assert( + JS_FOR_EACH_TRACEKIND(IS_SAME_TYPE_OR) + mozilla::IsSame<T, JS::Value>::value || + mozilla::IsSame<T, jsid>::value || + mozilla::IsSame<T, TaggedProto>::value, + "Only the base cell layout types are allowed into marking/tracing internals"); +#undef IS_SAME_TYPE_OR + if (trc->isMarkingTracer()) + return DoMarking(static_cast<GCMarker*>(trc), *thingp); + if (trc->isTenuringTracer()) + return static_cast<TenuringTracer*>(trc)->traverse(thingp); + MOZ_ASSERT(trc->isCallbackTracer()); + DoCallback(trc->asCallbackTracer(), thingp, name); +} + + +/*** GC Marking Interface *************************************************************************/ + +namespace js { + +typedef bool HasNoImplicitEdgesType; + +template <typename T> +struct ImplicitEdgeHolderType { + typedef HasNoImplicitEdgesType Type; +}; + +// For now, we only handle JSObject* and JSScript* keys, but the linear time +// algorithm can be easily extended by adding in more types here, then making +// GCMarker::traverse<T> call markPotentialEphemeronKey. +template <> +struct ImplicitEdgeHolderType<JSObject*> { + typedef JSObject* Type; +}; + +template <> +struct ImplicitEdgeHolderType<JSScript*> { + typedef JSScript* Type; +}; + +void +GCMarker::markEphemeronValues(gc::Cell* markedCell, WeakEntryVector& values) +{ + size_t initialLen = values.length(); + for (size_t i = 0; i < initialLen; i++) + values[i].weakmap->traceEntry(this, markedCell, values[i].key); + + // The vector should not be appended to during iteration because the key is + // already marked, and even in cases where we have a multipart key, we + // should only be inserting entries for the unmarked portions. + MOZ_ASSERT(values.length() == initialLen); +} + +template <typename T> +void +GCMarker::markImplicitEdgesHelper(T markedThing) +{ + if (!isWeakMarkingTracer()) + return; + + Zone* zone = gc::TenuredCell::fromPointer(markedThing)->zone(); + MOZ_ASSERT(zone->isGCMarking()); + MOZ_ASSERT(!zone->isGCSweeping()); + + auto p = zone->gcWeakKeys.get(JS::GCCellPtr(markedThing)); + if (!p) + return; + WeakEntryVector& markables = p->value; + + markEphemeronValues(markedThing, markables); + markables.clear(); // If key address is reused, it should do nothing +} + +template <> +void +GCMarker::markImplicitEdgesHelper(HasNoImplicitEdgesType) +{ +} + +template <typename T> +void +GCMarker::markImplicitEdges(T* thing) +{ + markImplicitEdgesHelper<typename ImplicitEdgeHolderType<T*>::Type>(thing); +} + +} // namespace js + +template <typename T> +static inline bool +MustSkipMarking(GCMarker* gcmarker, T thing) +{ + // Don't trace things that are owned by another runtime. + if (IsOwnedByOtherRuntime(gcmarker->runtime(), thing)) + return true; + + // Don't mark things outside a zone if we are in a per-zone GC. + return !thing->zone()->isGCMarking(); +} + +template <> +bool +MustSkipMarking<JSObject*>(GCMarker* gcmarker, JSObject* obj) +{ + // Don't trace things that are owned by another runtime. + if (IsOwnedByOtherRuntime(gcmarker->runtime(), obj)) + return true; + + // We may mark a Nursery thing outside the context of the + // MinorCollectionTracer because of a pre-barrier. The pre-barrier is not + // needed in this case because we perform a minor collection before each + // incremental slice. + if (IsInsideNursery(obj)) + return true; + + // Don't mark things outside a zone if we are in a per-zone GC. It is + // faster to check our own arena, which we can do since we know that + // the object is tenured. + return !TenuredCell::fromPointer(obj)->zone()->isGCMarking(); +} + +template <typename T> +void +DoMarking(GCMarker* gcmarker, T* thing) +{ + // Do per-type marking precondition checks. + if (MustSkipMarking(gcmarker, thing)) + return; + + CheckTracedThing(gcmarker, thing); + gcmarker->traverse(thing); + + // Mark the compartment as live. + SetMaybeAliveFlag(thing); +} + +template <typename S> +struct DoMarkingFunctor : public VoidDefaultAdaptor<S> { + template <typename T> void operator()(T* t, GCMarker* gcmarker) { DoMarking(gcmarker, t); } +}; + +template <typename T> +void +DoMarking(GCMarker* gcmarker, const T& thing) +{ + DispatchTyped(DoMarkingFunctor<T>(), thing, gcmarker); +} + +template <typename T> +void +NoteWeakEdge(GCMarker* gcmarker, T** thingp) +{ + // Do per-type marking precondition checks. + if (MustSkipMarking(gcmarker, *thingp)) + return; + + CheckTracedThing(gcmarker, *thingp); + + // If the target is already marked, there's no need to store the edge. + if (IsMarkedUnbarriered(gcmarker->runtime(), thingp)) + return; + + gcmarker->noteWeakEdge(thingp); +} + +template <typename T> +void +NoteWeakEdge(GCMarker* gcmarker, T* thingp) +{ + MOZ_CRASH("the gc does not support tagged pointers as weak edges"); +} + +template <typename T> +void +js::GCMarker::noteWeakEdge(T* edge) +{ + static_assert(IsBaseOf<Cell, typename mozilla::RemovePointer<T>::Type>::value, + "edge must point to a GC pointer"); + MOZ_ASSERT((*edge)->isTenured()); + + // Note: we really want the *source* Zone here. The edge may start in a + // non-gc heap location, however, so we use the fact that cross-zone weak + // references are not allowed and use the *target's* zone. + JS::Zone::WeakEdges &weakRefs = (*edge)->asTenured().zone()->gcWeakRefs; + AutoEnterOOMUnsafeRegion oomUnsafe; + if (!weakRefs.append(reinterpret_cast<TenuredCell**>(edge))) + oomUnsafe.crash("Failed to record a weak edge for sweeping."); +} + +// The simplest traversal calls out to the fully generic traceChildren function +// to visit the child edges. In the absence of other traversal mechanisms, this +// function will rapidly grow the stack past its bounds and crash the process. +// Thus, this generic tracing should only be used in cases where subsequent +// tracing will not recurse. +template <typename T> +void +js::GCMarker::markAndTraceChildren(T* thing) +{ + if (ThingIsPermanentAtomOrWellKnownSymbol(thing)) + return; + if (mark(thing)) + thing->traceChildren(this); +} +namespace js { +template <> void GCMarker::traverse(BaseShape* thing) { markAndTraceChildren(thing); } +template <> void GCMarker::traverse(JS::Symbol* thing) { markAndTraceChildren(thing); } +} // namespace js + +// Strings, LazyScripts, Shapes, and Scopes are extremely common, but have +// simple patterns of recursion. We traverse trees of these edges immediately, +// with aggressive, manual inlining, implemented by eagerlyTraceChildren. +template <typename T> +void +js::GCMarker::markAndScan(T* thing) +{ + if (ThingIsPermanentAtomOrWellKnownSymbol(thing)) + return; + if (mark(thing)) + eagerlyMarkChildren(thing); +} +namespace js { +template <> void GCMarker::traverse(JSString* thing) { markAndScan(thing); } +template <> void GCMarker::traverse(LazyScript* thing) { markAndScan(thing); } +template <> void GCMarker::traverse(Shape* thing) { markAndScan(thing); } +template <> void GCMarker::traverse(js::Scope* thing) { markAndScan(thing); } +} // namespace js + +// Object and ObjectGroup are extremely common and can contain arbitrarily +// nested graphs, so are not trivially inlined. In this case we use a mark +// stack to control recursion. JitCode shares none of these properties, but is +// included for historical reasons. JSScript normally cannot recurse, but may +// be used as a weakmap key and thereby recurse into weakmapped values. +template <typename T> +void +js::GCMarker::markAndPush(StackTag tag, T* thing) +{ + if (!mark(thing)) + return; + pushTaggedPtr(tag, thing); + markImplicitEdges(thing); +} +namespace js { +template <> void GCMarker::traverse(JSObject* thing) { markAndPush(ObjectTag, thing); } +template <> void GCMarker::traverse(ObjectGroup* thing) { markAndPush(GroupTag, thing); } +template <> void GCMarker::traverse(jit::JitCode* thing) { markAndPush(JitCodeTag, thing); } +template <> void GCMarker::traverse(JSScript* thing) { markAndPush(ScriptTag, thing); } +} // namespace js + +namespace js { +template <> +void +GCMarker::traverse(AccessorShape* thing) { + MOZ_CRASH("AccessorShape must be marked as a Shape"); +} +} // namespace js + +template <typename S, typename T> +static void +CheckTraversedEdge(S source, T* target) +{ + // Atoms and Symbols do not have or mark their internal pointers, respectively. + MOZ_ASSERT(!ThingIsPermanentAtomOrWellKnownSymbol(source)); + + // The Zones must match, unless the target is an atom. + MOZ_ASSERT_IF(!ThingIsPermanentAtomOrWellKnownSymbol(target), + target->zone()->isAtomsZone() || target->zone() == source->zone()); + + // Atoms and Symbols do not have access to a compartment pointer, or we'd need + // to adjust the subsequent check to catch that case. + MOZ_ASSERT_IF(ThingIsPermanentAtomOrWellKnownSymbol(target), !target->maybeCompartment()); + MOZ_ASSERT_IF(target->zoneFromAnyThread()->isAtomsZone(), !target->maybeCompartment()); + // If we have access to a compartment pointer for both things, they must match. + MOZ_ASSERT_IF(source->maybeCompartment() && target->maybeCompartment(), + source->maybeCompartment() == target->maybeCompartment()); +} + +template <typename S, typename T> +void +js::GCMarker::traverseEdge(S source, T* target) +{ + CheckTraversedEdge(source, target); + traverse(target); +} + +template <typename V, typename S> struct TraverseEdgeFunctor : public VoidDefaultAdaptor<V> { + template <typename T> void operator()(T t, GCMarker* gcmarker, S s) { + return gcmarker->traverseEdge(s, t); + } +}; + +template <typename S, typename T> +void +js::GCMarker::traverseEdge(S source, const T& thing) +{ + DispatchTyped(TraverseEdgeFunctor<T, S>(), thing, this, source); +} + +template <typename T> +bool +js::GCMarker::mark(T* thing) +{ + AssertZoneIsMarking(thing); + MOZ_ASSERT(!IsInsideNursery(gc::TenuredCell::fromPointer(thing))); + return gc::ParticipatesInCC<T>::value + ? gc::TenuredCell::fromPointer(thing)->markIfUnmarked(markColor()) + : gc::TenuredCell::fromPointer(thing)->markIfUnmarked(gc::BLACK); +} + + +/*** Inline, Eager GC Marking *********************************************************************/ + +// Each of the eager, inline marking paths is directly preceeded by the +// out-of-line, generic tracing code for comparison. Both paths must end up +// traversing equivalent subgraphs. + +void +LazyScript::traceChildren(JSTracer* trc) +{ + if (script_) + TraceWeakEdge(trc, &script_, "script"); + + if (function_) + TraceEdge(trc, &function_, "function"); + + if (sourceObject_) + TraceEdge(trc, &sourceObject_, "sourceObject"); + + if (enclosingScope_) + TraceEdge(trc, &enclosingScope_, "enclosingScope"); + + // We rely on the fact that atoms are always tenured. + JSAtom** closedOverBindings = this->closedOverBindings(); + for (auto i : MakeRange(numClosedOverBindings())) { + if (closedOverBindings[i]) + TraceManuallyBarrieredEdge(trc, &closedOverBindings[i], "closedOverBinding"); + } + + GCPtrFunction* innerFunctions = this->innerFunctions(); + for (auto i : MakeRange(numInnerFunctions())) + TraceEdge(trc, &innerFunctions[i], "lazyScriptInnerFunction"); +} +inline void +js::GCMarker::eagerlyMarkChildren(LazyScript *thing) +{ + if (thing->script_) + noteWeakEdge(thing->script_.unsafeUnbarrieredForTracing()); + + if (thing->function_) + traverseEdge(thing, static_cast<JSObject*>(thing->function_)); + + if (thing->sourceObject_) + traverseEdge(thing, static_cast<JSObject*>(thing->sourceObject_)); + + if (thing->enclosingScope_) + traverseEdge(thing, static_cast<Scope*>(thing->enclosingScope_)); + + // We rely on the fact that atoms are always tenured. + JSAtom** closedOverBindings = thing->closedOverBindings(); + for (auto i : MakeRange(thing->numClosedOverBindings())) { + if (closedOverBindings[i]) + traverseEdge(thing, static_cast<JSString*>(closedOverBindings[i])); + } + + GCPtrFunction* innerFunctions = thing->innerFunctions(); + for (auto i : MakeRange(thing->numInnerFunctions())) + traverseEdge(thing, static_cast<JSObject*>(innerFunctions[i])); +} + +void +Shape::traceChildren(JSTracer* trc) +{ + TraceEdge(trc, &base_, "base"); + TraceEdge(trc, &propidRef(), "propid"); + if (parent) + TraceEdge(trc, &parent, "parent"); + + if (hasGetterObject()) + TraceManuallyBarrieredEdge(trc, &asAccessorShape().getterObj, "getter"); + if (hasSetterObject()) + TraceManuallyBarrieredEdge(trc, &asAccessorShape().setterObj, "setter"); +} +inline void +js::GCMarker::eagerlyMarkChildren(Shape* shape) +{ + MOZ_ASSERT(shape->isMarked(this->markColor())); + do { + // Special case: if a base shape has a shape table then all its pointers + // must point to this shape or an anscestor. Since these pointers will + // be traced by this loop they do not need to be traced here as well. + BaseShape* base = shape->base(); + CheckTraversedEdge(shape, base); + if (mark(base)) { + MOZ_ASSERT(base->canSkipMarkingShapeTable(shape)); + base->traceChildrenSkipShapeTable(this); + } + + traverseEdge(shape, shape->propidRef().get()); + + // When triggered between slices on belhalf of a barrier, these + // objects may reside in the nursery, so require an extra check. + // FIXME: Bug 1157967 - remove the isTenured checks. + if (shape->hasGetterObject() && shape->getterObject()->isTenured()) + traverseEdge(shape, shape->getterObject()); + if (shape->hasSetterObject() && shape->setterObject()->isTenured()) + traverseEdge(shape, shape->setterObject()); + + shape = shape->previous(); + } while (shape && mark(shape)); +} + +void +JSString::traceChildren(JSTracer* trc) +{ + if (hasBase()) + traceBase(trc); + else if (isRope()) + asRope().traceChildren(trc); +} +inline void +GCMarker::eagerlyMarkChildren(JSString* str) +{ + if (str->isLinear()) + eagerlyMarkChildren(&str->asLinear()); + else + eagerlyMarkChildren(&str->asRope()); +} + +void +JSString::traceBase(JSTracer* trc) +{ + MOZ_ASSERT(hasBase()); + TraceManuallyBarrieredEdge(trc, &d.s.u3.base, "base"); +} +inline void +js::GCMarker::eagerlyMarkChildren(JSLinearString* linearStr) +{ + AssertZoneIsMarking(linearStr); + MOZ_ASSERT(linearStr->isMarked()); + MOZ_ASSERT(linearStr->JSString::isLinear()); + + // Use iterative marking to avoid blowing out the stack. + while (linearStr->hasBase()) { + linearStr = linearStr->base(); + MOZ_ASSERT(linearStr->JSString::isLinear()); + if (linearStr->isPermanentAtom()) + break; + AssertZoneIsMarking(linearStr); + if (!mark(static_cast<JSString*>(linearStr))) + break; + } +} + +void +JSRope::traceChildren(JSTracer* trc) { + js::TraceManuallyBarrieredEdge(trc, &d.s.u2.left, "left child"); + js::TraceManuallyBarrieredEdge(trc, &d.s.u3.right, "right child"); +} +inline void +js::GCMarker::eagerlyMarkChildren(JSRope* rope) +{ + // This function tries to scan the whole rope tree using the marking stack + // as temporary storage. If that becomes full, the unscanned ropes are + // added to the delayed marking list. When the function returns, the + // marking stack is at the same depth as it was on entry. This way we avoid + // using tags when pushing ropes to the stack as ropes never leak to other + // users of the stack. This also assumes that a rope can only point to + // other ropes or linear strings, it cannot refer to GC things of other + // types. + ptrdiff_t savedPos = stack.position(); + JS_DIAGNOSTICS_ASSERT(rope->getTraceKind() == JS::TraceKind::String); +#ifdef JS_DEBUG + static const size_t DEEP_ROPE_THRESHOLD = 100000; + static const size_t ROPE_CYCLE_HISTORY = 100; + DebugOnly<size_t> ropeDepth = 0; + JSRope* history[ROPE_CYCLE_HISTORY]; +#endif + while (true) { +#ifdef JS_DEBUG + if (++ropeDepth >= DEEP_ROPE_THRESHOLD) { + // Bug 1011786 comment 294 - detect cyclic ropes. There are some + // legitimate deep ropes, at least in tests. So if we hit a deep + // rope, start recording the nodes we visit and check whether we + // repeat. But do it on a finite window size W so that we're not + // scanning the full history for every node. And only check every + // Wth push, to add only constant overhead per node. This will only + // catch cycles of size up to W (but it seems most likely that any + // cycles will be size 1 or maybe 2.) + if ((ropeDepth > DEEP_ROPE_THRESHOLD + ROPE_CYCLE_HISTORY) && + (ropeDepth % ROPE_CYCLE_HISTORY) == 0) + { + for (size_t i = 0; i < ROPE_CYCLE_HISTORY; i++) + MOZ_ASSERT(history[i] != rope, "cycle detected in rope"); + } + history[ropeDepth % ROPE_CYCLE_HISTORY] = rope; + } +#endif + + JS_DIAGNOSTICS_ASSERT(rope->getTraceKind() == JS::TraceKind::String); + JS_DIAGNOSTICS_ASSERT(rope->JSString::isRope()); + AssertZoneIsMarking(rope); + MOZ_ASSERT(rope->isMarked()); + JSRope* next = nullptr; + + JSString* right = rope->rightChild(); + if (!right->isPermanentAtom() && + mark(right)) + { + if (right->isLinear()) + eagerlyMarkChildren(&right->asLinear()); + else + next = &right->asRope(); + } + + JSString* left = rope->leftChild(); + if (!left->isPermanentAtom() && + mark(left)) + { + if (left->isLinear()) { + eagerlyMarkChildren(&left->asLinear()); + } else { + // When both children are ropes, set aside the right one to + // scan it later. + if (next && !stack.push(reinterpret_cast<uintptr_t>(next))) + delayMarkingChildren(next); + next = &left->asRope(); + } + } + if (next) { + rope = next; + } else if (savedPos != stack.position()) { + MOZ_ASSERT(savedPos < stack.position()); + rope = reinterpret_cast<JSRope*>(stack.pop()); + } else { + break; + } + } + MOZ_ASSERT(savedPos == stack.position()); +} + +static inline void +TraceBindingNames(JSTracer* trc, BindingName* names, uint32_t length) +{ + for (uint32_t i = 0; i < length; i++) { + JSAtom* name = names[i].name(); + MOZ_ASSERT(name); + TraceManuallyBarrieredEdge(trc, &name, "scope name"); + } +}; +static inline void +TraceNullableBindingNames(JSTracer* trc, BindingName* names, uint32_t length) +{ + for (uint32_t i = 0; i < length; i++) { + if (JSAtom* name = names[i].name()) + TraceManuallyBarrieredEdge(trc, &name, "scope name"); + } +}; +void +BindingName::trace(JSTracer* trc) +{ + if (JSAtom* atom = name()) + TraceManuallyBarrieredEdge(trc, &atom, "binding name"); +} +void +BindingIter::trace(JSTracer* trc) +{ + TraceNullableBindingNames(trc, names_, length_); +} +void +LexicalScope::Data::trace(JSTracer* trc) +{ + TraceBindingNames(trc, names, length); +} +void +FunctionScope::Data::trace(JSTracer* trc) +{ + TraceNullableEdge(trc, &canonicalFunction, "scope canonical function"); + TraceNullableBindingNames(trc, names, length); +} +void +VarScope::Data::trace(JSTracer* trc) +{ + TraceBindingNames(trc, names, length); +} +void +GlobalScope::Data::trace(JSTracer* trc) +{ + TraceBindingNames(trc, names, length); +} +void +EvalScope::Data::trace(JSTracer* trc) +{ + TraceBindingNames(trc, names, length); +} +void +ModuleScope::Data::trace(JSTracer* trc) +{ + TraceNullableEdge(trc, &module, "scope module"); + TraceBindingNames(trc, names, length); +} +void +Scope::traceChildren(JSTracer* trc) +{ + TraceNullableEdge(trc, &enclosing_, "scope enclosing"); + TraceNullableEdge(trc, &environmentShape_, "scope env shape"); + switch (kind_) { + case ScopeKind::Function: + reinterpret_cast<FunctionScope::Data*>(data_)->trace(trc); + break; + case ScopeKind::FunctionBodyVar: + case ScopeKind::ParameterExpressionVar: + reinterpret_cast<VarScope::Data*>(data_)->trace(trc); + break; + case ScopeKind::Lexical: + case ScopeKind::SimpleCatch: + case ScopeKind::Catch: + case ScopeKind::NamedLambda: + case ScopeKind::StrictNamedLambda: + reinterpret_cast<LexicalScope::Data*>(data_)->trace(trc); + break; + case ScopeKind::Global: + case ScopeKind::NonSyntactic: + reinterpret_cast<GlobalScope::Data*>(data_)->trace(trc); + break; + case ScopeKind::Eval: + case ScopeKind::StrictEval: + reinterpret_cast<EvalScope::Data*>(data_)->trace(trc); + break; + case ScopeKind::Module: + reinterpret_cast<ModuleScope::Data*>(data_)->trace(trc); + break; + case ScopeKind::With: + break; + } +} +inline void +js::GCMarker::eagerlyMarkChildren(Scope* scope) +{ + if (scope->enclosing_) + traverseEdge(scope, static_cast<Scope*>(scope->enclosing_)); + if (scope->environmentShape_) + traverseEdge(scope, static_cast<Shape*>(scope->environmentShape_)); + BindingName* names = nullptr; + uint32_t length = 0; + switch (scope->kind_) { + case ScopeKind::Function: { + FunctionScope::Data* data = reinterpret_cast<FunctionScope::Data*>(scope->data_); + traverseEdge(scope, static_cast<JSObject*>(data->canonicalFunction)); + names = data->names; + length = data->length; + break; + } + + case ScopeKind::FunctionBodyVar: + case ScopeKind::ParameterExpressionVar: { + VarScope::Data* data = reinterpret_cast<VarScope::Data*>(scope->data_); + names = data->names; + length = data->length; + break; + } + + case ScopeKind::Lexical: + case ScopeKind::SimpleCatch: + case ScopeKind::Catch: + case ScopeKind::NamedLambda: + case ScopeKind::StrictNamedLambda: { + LexicalScope::Data* data = reinterpret_cast<LexicalScope::Data*>(scope->data_); + names = data->names; + length = data->length; + break; + } + + case ScopeKind::Global: + case ScopeKind::NonSyntactic: { + GlobalScope::Data* data = reinterpret_cast<GlobalScope::Data*>(scope->data_); + names = data->names; + length = data->length; + break; + } + + case ScopeKind::Eval: + case ScopeKind::StrictEval: { + EvalScope::Data* data = reinterpret_cast<EvalScope::Data*>(scope->data_); + names = data->names; + length = data->length; + break; + } + + case ScopeKind::Module: { + ModuleScope::Data* data = reinterpret_cast<ModuleScope::Data*>(scope->data_); + traverseEdge(scope, static_cast<JSObject*>(data->module)); + names = data->names; + length = data->length; + break; + } + + case ScopeKind::With: + break; + } + if (scope->kind_ == ScopeKind::Function) { + for (uint32_t i = 0; i < length; i++) { + if (JSAtom* name = names[i].name()) + traverseEdge(scope, static_cast<JSString*>(name)); + } + } else { + for (uint32_t i = 0; i < length; i++) + traverseEdge(scope, static_cast<JSString*>(names[i].name())); + } +} + +void +js::ObjectGroup::traceChildren(JSTracer* trc) +{ + unsigned count = getPropertyCount(); + for (unsigned i = 0; i < count; i++) { + if (ObjectGroup::Property* prop = getProperty(i)) + TraceEdge(trc, &prop->id, "group_property"); + } + + if (proto().isObject()) + TraceEdge(trc, &proto(), "group_proto"); + + if (trc->isMarkingTracer()) + compartment()->mark(); + + if (JSObject* global = compartment()->unsafeUnbarrieredMaybeGlobal()) + TraceManuallyBarrieredEdge(trc, &global, "group_global"); + + + if (newScript()) + newScript()->trace(trc); + + if (maybePreliminaryObjects()) + maybePreliminaryObjects()->trace(trc); + + if (maybeUnboxedLayout()) + unboxedLayout().trace(trc); + + if (ObjectGroup* unboxedGroup = maybeOriginalUnboxedGroup()) { + TraceManuallyBarrieredEdge(trc, &unboxedGroup, "group_original_unboxed_group"); + setOriginalUnboxedGroup(unboxedGroup); + } + + if (JSObject* descr = maybeTypeDescr()) { + TraceManuallyBarrieredEdge(trc, &descr, "group_type_descr"); + setTypeDescr(&descr->as<TypeDescr>()); + } + + if (JSObject* fun = maybeInterpretedFunction()) { + TraceManuallyBarrieredEdge(trc, &fun, "group_function"); + setInterpretedFunction(&fun->as<JSFunction>()); + } +} +void +js::GCMarker::lazilyMarkChildren(ObjectGroup* group) +{ + unsigned count = group->getPropertyCount(); + for (unsigned i = 0; i < count; i++) { + if (ObjectGroup::Property* prop = group->getProperty(i)) + traverseEdge(group, prop->id.get()); + } + + if (group->proto().isObject()) + traverseEdge(group, group->proto().toObject()); + + group->compartment()->mark(); + + if (GlobalObject* global = group->compartment()->unsafeUnbarrieredMaybeGlobal()) + traverseEdge(group, static_cast<JSObject*>(global)); + + if (group->newScript()) + group->newScript()->trace(this); + + if (group->maybePreliminaryObjects()) + group->maybePreliminaryObjects()->trace(this); + + if (group->maybeUnboxedLayout()) + group->unboxedLayout().trace(this); + + if (ObjectGroup* unboxedGroup = group->maybeOriginalUnboxedGroup()) + traverseEdge(group, unboxedGroup); + + if (TypeDescr* descr = group->maybeTypeDescr()) + traverseEdge(group, static_cast<JSObject*>(descr)); + + if (JSFunction* fun = group->maybeInterpretedFunction()) + traverseEdge(group, static_cast<JSObject*>(fun)); +} + +struct TraverseObjectFunctor +{ + template <typename T> + void operator()(T* thing, GCMarker* gcmarker, JSObject* src) { + gcmarker->traverseEdge(src, *thing); + } +}; + +// Call the trace hook set on the object, if present. If further tracing of +// NativeObject fields is required, this will return the native object. +enum class CheckGeneration { DoChecks, NoChecks}; +template <typename Functor, typename... Args> +static inline NativeObject* +CallTraceHook(Functor f, JSTracer* trc, JSObject* obj, CheckGeneration check, Args&&... args) +{ + const Class* clasp = obj->getClass(); + MOZ_ASSERT(clasp); + MOZ_ASSERT(obj->isNative() == clasp->isNative()); + + if (!clasp->hasTrace()) + return &obj->as<NativeObject>(); + + if (clasp->isTrace(InlineTypedObject::obj_trace)) { + Shape** pshape = obj->as<InlineTypedObject>().addressOfShapeFromGC(); + f(pshape, mozilla::Forward<Args>(args)...); + + InlineTypedObject& tobj = obj->as<InlineTypedObject>(); + if (tobj.typeDescr().hasTraceList()) { + VisitTraceList(f, tobj.typeDescr().traceList(), tobj.inlineTypedMemForGC(), + mozilla::Forward<Args>(args)...); + } + + return nullptr; + } + + if (clasp == &UnboxedPlainObject::class_) { + JSObject** pexpando = obj->as<UnboxedPlainObject>().addressOfExpando(); + if (*pexpando) + f(pexpando, mozilla::Forward<Args>(args)...); + + UnboxedPlainObject& unboxed = obj->as<UnboxedPlainObject>(); + const UnboxedLayout& layout = check == CheckGeneration::DoChecks + ? unboxed.layout() + : unboxed.layoutDontCheckGeneration(); + if (layout.traceList()) { + VisitTraceList(f, layout.traceList(), unboxed.data(), + mozilla::Forward<Args>(args)...); + } + + return nullptr; + } + + clasp->doTrace(trc, obj); + + if (!clasp->isNative()) + return nullptr; + return &obj->as<NativeObject>(); +} + +template <typename F, typename... Args> +static void +VisitTraceList(F f, const int32_t* traceList, uint8_t* memory, Args&&... args) +{ + while (*traceList != -1) { + f(reinterpret_cast<JSString**>(memory + *traceList), mozilla::Forward<Args>(args)...); + traceList++; + } + traceList++; + while (*traceList != -1) { + JSObject** objp = reinterpret_cast<JSObject**>(memory + *traceList); + if (*objp) + f(objp, mozilla::Forward<Args>(args)...); + traceList++; + } + traceList++; + while (*traceList != -1) { + f(reinterpret_cast<Value*>(memory + *traceList), mozilla::Forward<Args>(args)...); + traceList++; + } +} + + +/*** Mark-stack Marking ***************************************************************************/ + +bool +GCMarker::drainMarkStack(SliceBudget& budget) +{ +#ifdef DEBUG + MOZ_ASSERT(!strictCompartmentChecking); + strictCompartmentChecking = true; + auto acc = mozilla::MakeScopeExit([&] {strictCompartmentChecking = false;}); +#endif + + if (budget.isOverBudget()) + return false; + + for (;;) { + while (!stack.isEmpty()) { + processMarkStackTop(budget); + if (budget.isOverBudget()) { + saveValueRanges(); + return false; + } + } + + if (!hasDelayedChildren()) + break; + + /* + * Mark children of things that caused too deep recursion during the + * above tracing. Don't do this until we're done with everything + * else. + */ + if (!markDelayedChildren(budget)) { + saveValueRanges(); + return false; + } + } + + return true; +} + +inline static bool +ObjectDenseElementsMayBeMarkable(NativeObject* nobj) +{ + /* + * For arrays that are large enough it's worth checking the type information + * to see if the object's elements contain any GC pointers. If not, we + * don't need to trace them. + */ + const unsigned MinElementsLength = 32; + if (nobj->getDenseInitializedLength() < MinElementsLength || nobj->isSingleton()) + return true; + + ObjectGroup* group = nobj->group(); + if (group->needsSweep() || group->unknownProperties()) + return true; + + HeapTypeSet* typeSet = group->maybeGetProperty(JSID_VOID); + if (!typeSet) + return true; + + static const uint32_t flagMask = + TYPE_FLAG_STRING | TYPE_FLAG_SYMBOL | TYPE_FLAG_LAZYARGS | TYPE_FLAG_ANYOBJECT; + bool mayBeMarkable = typeSet->hasAnyFlag(flagMask) || typeSet->getObjectCount() != 0; + +#ifdef DEBUG + if (!mayBeMarkable) { + const Value* elements = nobj->getDenseElementsAllowCopyOnWrite(); + for (unsigned i = 0; i < nobj->getDenseInitializedLength(); i++) + MOZ_ASSERT(!elements[i].isMarkable()); + } +#endif + + return mayBeMarkable; +} + +inline void +GCMarker::processMarkStackTop(SliceBudget& budget) +{ + /* + * The function uses explicit goto and implements the scanning of the + * object directly. It allows to eliminate the tail recursion and + * significantly improve the marking performance, see bug 641025. + */ + HeapSlot* vp; + HeapSlot* end; + JSObject* obj; + + // Decode + uintptr_t addr = stack.pop(); + uintptr_t tag = addr & StackTagMask; + addr &= ~StackTagMask; + + // Dispatch + switch (tag) { + case ValueArrayTag: { + JS_STATIC_ASSERT(ValueArrayTag == 0); + MOZ_ASSERT(!(addr & CellMask)); + obj = reinterpret_cast<JSObject*>(addr); + uintptr_t addr2 = stack.pop(); + uintptr_t addr3 = stack.pop(); + MOZ_ASSERT(addr2 <= addr3); + MOZ_ASSERT((addr3 - addr2) % sizeof(Value) == 0); + vp = reinterpret_cast<HeapSlot*>(addr2); + end = reinterpret_cast<HeapSlot*>(addr3); + goto scan_value_array; + } + + case ObjectTag: { + obj = reinterpret_cast<JSObject*>(addr); + AssertZoneIsMarking(obj); + goto scan_obj; + } + + case GroupTag: { + return lazilyMarkChildren(reinterpret_cast<ObjectGroup*>(addr)); + } + + case JitCodeTag: { + return reinterpret_cast<jit::JitCode*>(addr)->traceChildren(this); + } + + case ScriptTag: { + return reinterpret_cast<JSScript*>(addr)->traceChildren(this); + } + + case SavedValueArrayTag: { + MOZ_ASSERT(!(addr & CellMask)); + JSObject* obj = reinterpret_cast<JSObject*>(addr); + HeapSlot* vp; + HeapSlot* end; + if (restoreValueArray(obj, (void**)&vp, (void**)&end)) + pushValueArray(&obj->as<NativeObject>(), vp, end); + else + repush(obj); + return; + } + + default: MOZ_CRASH("Invalid tag in mark stack"); + } + return; + + scan_value_array: + MOZ_ASSERT(vp <= end); + while (vp != end) { + budget.step(); + if (budget.isOverBudget()) { + pushValueArray(obj, vp, end); + return; + } + + const Value& v = *vp++; + if (v.isString()) { + traverseEdge(obj, v.toString()); + } else if (v.isObject()) { + JSObject* obj2 = &v.toObject(); + MOZ_ASSERT(obj->compartment() == obj2->compartment()); + if (mark(obj2)) { + // Save the rest of this value array for later and start scanning obj2's children. + pushValueArray(obj, vp, end); + obj = obj2; + goto scan_obj; + } + } else if (v.isSymbol()) { + traverseEdge(obj, v.toSymbol()); + } else if (v.isPrivateGCThing()) { + traverseEdge(obj, v.toGCCellPtr()); + } + } + return; + + scan_obj: + { + AssertZoneIsMarking(obj); + + budget.step(); + if (budget.isOverBudget()) { + repush(obj); + return; + } + + markImplicitEdges(obj); + ObjectGroup* group = obj->groupFromGC(); + traverseEdge(obj, group); + + NativeObject *nobj = CallTraceHook(TraverseObjectFunctor(), this, obj, + CheckGeneration::DoChecks, this, obj); + if (!nobj) + return; + + Shape* shape = nobj->lastProperty(); + traverseEdge(obj, shape); + + unsigned nslots = nobj->slotSpan(); + + do { + if (nobj->hasEmptyElements()) + break; + + if (nobj->denseElementsAreCopyOnWrite()) { + JSObject* owner = nobj->getElementsHeader()->ownerObject(); + if (owner != nobj) { + traverseEdge(obj, owner); + break; + } + } + + if (!ObjectDenseElementsMayBeMarkable(nobj)) + break; + + vp = nobj->getDenseElementsAllowCopyOnWrite(); + end = vp + nobj->getDenseInitializedLength(); + + if (!nslots) + goto scan_value_array; + pushValueArray(nobj, vp, end); + } while (false); + + vp = nobj->fixedSlots(); + if (nobj->slots_) { + unsigned nfixed = nobj->numFixedSlots(); + if (nslots > nfixed) { + pushValueArray(nobj, vp, vp + nfixed); + vp = nobj->slots_; + end = vp + (nslots - nfixed); + goto scan_value_array; + } + } + MOZ_ASSERT(nslots <= nobj->numFixedSlots()); + end = vp + nslots; + goto scan_value_array; + } +} + +struct SlotArrayLayout +{ + union { + HeapSlot* end; + uintptr_t kind; + }; + union { + HeapSlot* start; + uintptr_t index; + }; + NativeObject* obj; + + static void staticAsserts() { + /* This should have the same layout as three mark stack items. */ + JS_STATIC_ASSERT(sizeof(SlotArrayLayout) == 3 * sizeof(uintptr_t)); + } +}; + +/* + * During incremental GC, we return from drainMarkStack without having processed + * the entire stack. At that point, JS code can run and reallocate slot arrays + * that are stored on the stack. To prevent this from happening, we replace all + * ValueArrayTag stack items with SavedValueArrayTag. In the latter, slots + * pointers are replaced with slot indexes, and slot array end pointers are + * replaced with the kind of index (properties vs. elements). + */ +void +GCMarker::saveValueRanges() +{ + for (uintptr_t* p = stack.tos_; p > stack.stack_; ) { + uintptr_t tag = *--p & StackTagMask; + if (tag == ValueArrayTag) { + *p &= ~StackTagMask; + p -= 2; + SlotArrayLayout* arr = reinterpret_cast<SlotArrayLayout*>(p); + NativeObject* obj = arr->obj; + MOZ_ASSERT(obj->isNative()); + + HeapSlot* vp = obj->getDenseElementsAllowCopyOnWrite(); + if (arr->end == vp + obj->getDenseInitializedLength()) { + MOZ_ASSERT(arr->start >= vp); + arr->index = arr->start - vp; + arr->kind = HeapSlot::Element; + } else { + HeapSlot* vp = obj->fixedSlots(); + unsigned nfixed = obj->numFixedSlots(); + if (arr->start == arr->end) { + arr->index = obj->slotSpan(); + } else if (arr->start >= vp && arr->start < vp + nfixed) { + MOZ_ASSERT(arr->end == vp + Min(nfixed, obj->slotSpan())); + arr->index = arr->start - vp; + } else { + MOZ_ASSERT(arr->start >= obj->slots_ && + arr->end == obj->slots_ + obj->slotSpan() - nfixed); + arr->index = (arr->start - obj->slots_) + nfixed; + } + arr->kind = HeapSlot::Slot; + } + p[2] |= SavedValueArrayTag; + } else if (tag == SavedValueArrayTag) { + p -= 2; + } + } +} + +bool +GCMarker::restoreValueArray(JSObject* objArg, void** vpp, void** endp) +{ + uintptr_t start = stack.pop(); + HeapSlot::Kind kind = (HeapSlot::Kind) stack.pop(); + + if (!objArg->isNative()) + return false; + NativeObject* obj = &objArg->as<NativeObject>(); + + if (kind == HeapSlot::Element) { + if (!obj->is<ArrayObject>()) + return false; + + uint32_t initlen = obj->getDenseInitializedLength(); + HeapSlot* vp = obj->getDenseElementsAllowCopyOnWrite(); + if (start < initlen) { + *vpp = vp + start; + *endp = vp + initlen; + } else { + /* The object shrunk, in which case no scanning is needed. */ + *vpp = *endp = vp; + } + } else { + MOZ_ASSERT(kind == HeapSlot::Slot); + HeapSlot* vp = obj->fixedSlots(); + unsigned nfixed = obj->numFixedSlots(); + unsigned nslots = obj->slotSpan(); + if (start < nslots) { + if (start < nfixed) { + *vpp = vp + start; + *endp = vp + Min(nfixed, nslots); + } else { + *vpp = obj->slots_ + start - nfixed; + *endp = obj->slots_ + nslots - nfixed; + } + } else { + /* The object shrunk, in which case no scanning is needed. */ + *vpp = *endp = vp; + } + } + + MOZ_ASSERT(*vpp <= *endp); + return true; +} + + +/*** Mark Stack ***********************************************************************************/ + +bool +MarkStack::init(JSGCMode gcMode) +{ + setBaseCapacity(gcMode); + + MOZ_ASSERT(!stack_); + uintptr_t* newStack = js_pod_malloc<uintptr_t>(baseCapacity_); + if (!newStack) + return false; + + setStack(newStack, 0, baseCapacity_); + return true; +} + +void +MarkStack::setBaseCapacity(JSGCMode mode) +{ + switch (mode) { + case JSGC_MODE_GLOBAL: + case JSGC_MODE_ZONE: + baseCapacity_ = NON_INCREMENTAL_MARK_STACK_BASE_CAPACITY; + break; + case JSGC_MODE_INCREMENTAL: + baseCapacity_ = INCREMENTAL_MARK_STACK_BASE_CAPACITY; + break; + default: + MOZ_CRASH("bad gc mode"); + } + + if (baseCapacity_ > maxCapacity_) + baseCapacity_ = maxCapacity_; +} + +void +MarkStack::setMaxCapacity(size_t maxCapacity) +{ + MOZ_ASSERT(maxCapacity != 0); + MOZ_ASSERT(isEmpty()); + maxCapacity_ = maxCapacity; + if (baseCapacity_ > maxCapacity_) + baseCapacity_ = maxCapacity_; + + reset(); +} + +void +MarkStack::reset() +{ + if (capacity() == baseCapacity_) { + // No size change; keep the current stack. + setStack(stack_, 0, baseCapacity_); + return; + } + + MOZ_ASSERT(baseCapacity_ != 0); + uintptr_t* newStack = (uintptr_t*)js_realloc(stack_, sizeof(uintptr_t) * baseCapacity_); + if (!newStack) { + // If the realloc fails, just keep using the existing stack; it's + // not ideal but better than failing. + newStack = stack_; + baseCapacity_ = capacity(); + } + setStack(newStack, 0, baseCapacity_); +} + +bool +MarkStack::enlarge(unsigned count) +{ + size_t newCapacity = Min(maxCapacity_, capacity() * 2); + if (newCapacity < capacity() + count) + return false; + + size_t tosIndex = position(); + + MOZ_ASSERT(newCapacity != 0); + uintptr_t* newStack = (uintptr_t*)js_realloc(stack_, sizeof(uintptr_t) * newCapacity); + if (!newStack) + return false; + + setStack(newStack, tosIndex, newCapacity); + return true; +} + +void +MarkStack::setGCMode(JSGCMode gcMode) +{ + // The mark stack won't be resized until the next call to reset(), but + // that will happen at the end of the next GC. + setBaseCapacity(gcMode); +} + +size_t +MarkStack::sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const +{ + return mallocSizeOf(stack_); +} + + +/*** GCMarker *************************************************************************************/ + +/* + * ExpandWeakMaps: the GC is recomputing the liveness of WeakMap entries by + * expanding each live WeakMap into its constituent key->value edges, a table + * of which will be consulted in a later phase whenever marking a potential + * key. + */ +GCMarker::GCMarker(JSRuntime* rt) + : JSTracer(rt, JSTracer::TracerKindTag::Marking, ExpandWeakMaps), + stack(size_t(-1)), + color(BLACK), + unmarkedArenaStackTop(nullptr) +#ifdef DEBUG + , markLaterArenas(0) + , started(false) + , strictCompartmentChecking(false) +#endif +{ +} + +bool +GCMarker::init(JSGCMode gcMode) +{ + return stack.init(gcMode); +} + +void +GCMarker::start() +{ +#ifdef DEBUG + MOZ_ASSERT(!started); + started = true; +#endif + color = BLACK; + linearWeakMarkingDisabled_ = false; + + MOZ_ASSERT(!unmarkedArenaStackTop); + MOZ_ASSERT(markLaterArenas == 0); +} + +void +GCMarker::stop() +{ +#ifdef DEBUG + MOZ_ASSERT(isDrained()); + + MOZ_ASSERT(started); + started = false; + + MOZ_ASSERT(!unmarkedArenaStackTop); + MOZ_ASSERT(markLaterArenas == 0); +#endif + + /* Free non-ballast stack memory. */ + stack.reset(); + AutoEnterOOMUnsafeRegion oomUnsafe; + for (GCZonesIter zone(runtime()); !zone.done(); zone.next()) { + if (!zone->gcWeakKeys.clear()) + oomUnsafe.crash("clearing weak keys in GCMarker::stop()"); + } +} + +void +GCMarker::reset() +{ + color = BLACK; + + stack.reset(); + MOZ_ASSERT(isMarkStackEmpty()); + + while (unmarkedArenaStackTop) { + Arena* arena = unmarkedArenaStackTop; + MOZ_ASSERT(arena->hasDelayedMarking); + MOZ_ASSERT(markLaterArenas); + unmarkedArenaStackTop = arena->getNextDelayedMarking(); + arena->unsetDelayedMarking(); + arena->markOverflow = 0; + arena->allocatedDuringIncremental = 0; +#ifdef DEBUG + markLaterArenas--; +#endif + } + MOZ_ASSERT(isDrained()); + MOZ_ASSERT(!markLaterArenas); +} + +void +GCMarker::enterWeakMarkingMode() +{ + MOZ_ASSERT(tag_ == TracerKindTag::Marking); + if (linearWeakMarkingDisabled_) + return; + + // During weak marking mode, we maintain a table mapping weak keys to + // entries in known-live weakmaps. Initialize it with the keys of marked + // weakmaps -- or more precisely, the keys of marked weakmaps that are + // mapped to not yet live values. (Once bug 1167452 implements incremental + // weakmap marking, this initialization step will become unnecessary, as + // the table will already hold all such keys.) + if (weakMapAction() == ExpandWeakMaps) { + tag_ = TracerKindTag::WeakMarking; + + for (GCZoneGroupIter zone(runtime()); !zone.done(); zone.next()) { + for (WeakMapBase* m : zone->gcWeakMapList) { + if (m->marked) + (void) m->traceEntries(this); + } + } + } +} + +void +GCMarker::leaveWeakMarkingMode() +{ + MOZ_ASSERT_IF(weakMapAction() == ExpandWeakMaps && !linearWeakMarkingDisabled_, + tag_ == TracerKindTag::WeakMarking); + tag_ = TracerKindTag::Marking; + + // Table is expensive to maintain when not in weak marking mode, so we'll + // rebuild it upon entry rather than allow it to contain stale data. + AutoEnterOOMUnsafeRegion oomUnsafe; + for (GCZonesIter zone(runtime()); !zone.done(); zone.next()) { + if (!zone->gcWeakKeys.clear()) + oomUnsafe.crash("clearing weak keys in GCMarker::leaveWeakMarkingMode()"); + } +} + +void +GCMarker::markDelayedChildren(Arena* arena) +{ + if (arena->markOverflow) { + bool always = arena->allocatedDuringIncremental; + arena->markOverflow = 0; + + for (ArenaCellIterUnderGC i(arena); !i.done(); i.next()) { + TenuredCell* t = i.getCell(); + if (always || t->isMarked()) { + t->markIfUnmarked(); + js::TraceChildren(this, t, MapAllocToTraceKind(arena->getAllocKind())); + } + } + } else { + MOZ_ASSERT(arena->allocatedDuringIncremental); + PushArena(this, arena); + } + arena->allocatedDuringIncremental = 0; + /* + * Note that during an incremental GC we may still be allocating into + * the arena. However, prepareForIncrementalGC sets the + * allocatedDuringIncremental flag if we continue marking. + */ +} + +bool +GCMarker::markDelayedChildren(SliceBudget& budget) +{ + GCRuntime& gc = runtime()->gc; + gcstats::AutoPhase ap(gc.stats, gc.state() == State::Mark, gcstats::PHASE_MARK_DELAYED); + + MOZ_ASSERT(unmarkedArenaStackTop); + do { + /* + * If marking gets delayed at the same arena again, we must repeat + * marking of its things. For that we pop arena from the stack and + * clear its hasDelayedMarking flag before we begin the marking. + */ + Arena* arena = unmarkedArenaStackTop; + MOZ_ASSERT(arena->hasDelayedMarking); + MOZ_ASSERT(markLaterArenas); + unmarkedArenaStackTop = arena->getNextDelayedMarking(); + arena->unsetDelayedMarking(); +#ifdef DEBUG + markLaterArenas--; +#endif + markDelayedChildren(arena); + + budget.step(150); + if (budget.isOverBudget()) + return false; + } while (unmarkedArenaStackTop); + MOZ_ASSERT(!markLaterArenas); + + return true; +} + +template<typename T> +static void +PushArenaTyped(GCMarker* gcmarker, Arena* arena) +{ + for (ArenaCellIterUnderGC i(arena); !i.done(); i.next()) + gcmarker->traverse(i.get<T>()); +} + +struct PushArenaFunctor { + template <typename T> void operator()(GCMarker* gcmarker, Arena* arena) { + PushArenaTyped<T>(gcmarker, arena); + } +}; + +void +gc::PushArena(GCMarker* gcmarker, Arena* arena) +{ + DispatchTraceKindTyped(PushArenaFunctor(), + MapAllocToTraceKind(arena->getAllocKind()), gcmarker, arena); +} + +#ifdef DEBUG +void +GCMarker::checkZone(void* p) +{ + MOZ_ASSERT(started); + DebugOnly<Cell*> cell = static_cast<Cell*>(p); + MOZ_ASSERT_IF(cell->isTenured(), cell->asTenured().zone()->isCollecting()); +} +#endif + +size_t +GCMarker::sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const +{ + size_t size = stack.sizeOfExcludingThis(mallocSizeOf); + for (ZonesIter zone(runtime(), WithAtoms); !zone.done(); zone.next()) + size += zone->gcGrayRoots.sizeOfExcludingThis(mallocSizeOf); + return size; +} + + +/*** Tenuring Tracer *****************************************************************************/ + +namespace js { +template <typename T> +void +TenuringTracer::traverse(T** tp) +{ +} + +template <> +void +TenuringTracer::traverse(JSObject** objp) +{ + // We only ever visit the internals of objects after moving them to tenured. + MOZ_ASSERT(!nursery().isInside(objp)); + + if (IsInsideNursery(*objp) && !nursery().getForwardedPointer(objp)) + *objp = moveToTenured(*objp); +} + +template <typename S> +struct TenuringTraversalFunctor : public IdentityDefaultAdaptor<S> { + template <typename T> S operator()(T* t, TenuringTracer* trc) { + trc->traverse(&t); + return js::gc::RewrapTaggedPointer<S, T>::wrap(t); + } +}; + +template <typename T> +void +TenuringTracer::traverse(T* thingp) +{ + *thingp = DispatchTyped(TenuringTraversalFunctor<T>(), *thingp, this); +} +} // namespace js + +template <typename T> +void +js::gc::StoreBuffer::MonoTypeBuffer<T>::trace(StoreBuffer* owner, TenuringTracer& mover) +{ + mozilla::ReentrancyGuard g(*owner); + MOZ_ASSERT(owner->isEnabled()); + MOZ_ASSERT(stores_.initialized()); + if (last_) + last_.trace(mover); + for (typename StoreSet::Range r = stores_.all(); !r.empty(); r.popFront()) + r.front().trace(mover); +} + +namespace js { +namespace gc { +template void +StoreBuffer::MonoTypeBuffer<StoreBuffer::ValueEdge>::trace(StoreBuffer*, TenuringTracer&); +template void +StoreBuffer::MonoTypeBuffer<StoreBuffer::SlotsEdge>::trace(StoreBuffer*, TenuringTracer&); +template void +StoreBuffer::MonoTypeBuffer<StoreBuffer::CellPtrEdge>::trace(StoreBuffer*, TenuringTracer&); +} // namespace gc +} // namespace js + +void +js::gc::StoreBuffer::SlotsEdge::trace(TenuringTracer& mover) const +{ + NativeObject* obj = object(); + + // Beware JSObject::swap exchanging a native object for a non-native one. + if (!obj->isNative()) + return; + + if (IsInsideNursery(obj)) + return; + + if (kind() == ElementKind) { + int32_t initLen = obj->getDenseInitializedLength(); + int32_t clampedStart = Min(start_, initLen); + int32_t clampedEnd = Min(start_ + count_, initLen); + mover.traceSlots(static_cast<HeapSlot*>(obj->getDenseElements() + clampedStart) + ->unsafeUnbarrieredForTracing(), clampedEnd - clampedStart); + } else { + int32_t start = Min(uint32_t(start_), obj->slotSpan()); + int32_t end = Min(uint32_t(start_) + count_, obj->slotSpan()); + MOZ_ASSERT(end >= start); + mover.traceObjectSlots(obj, start, end - start); + } +} + +static inline void +TraceWholeCell(TenuringTracer& mover, JSObject* object) +{ + mover.traceObject(object); + + // Additionally trace the expando object attached to any unboxed plain + // objects. Baseline and Ion can write properties to the expando while + // only adding a post barrier to the owning unboxed object. Note that + // it isn't possible for a nursery unboxed object to have a tenured + // expando, so that adding a post barrier on the original object will + // capture any tenured->nursery edges in the expando as well. + + if (object->is<UnboxedPlainObject>()) { + if (UnboxedExpandoObject* expando = object->as<UnboxedPlainObject>().maybeExpando()) + expando->traceChildren(&mover); + } +} + +static inline void +TraceWholeCell(TenuringTracer& mover, JSScript* script) +{ + script->traceChildren(&mover); +} + +static inline void +TraceWholeCell(TenuringTracer& mover, jit::JitCode* jitcode) +{ + jitcode->traceChildren(&mover); +} + +template <typename T> +static void +TraceBufferedCells(TenuringTracer& mover, Arena* arena, ArenaCellSet* cells) +{ + for (size_t i = 0; i < ArenaCellCount; i++) { + if (cells->hasCell(i)) { + auto cell = reinterpret_cast<T*>(uintptr_t(arena) + CellSize * i); + TraceWholeCell(mover, cell); + } + } +} + +void +js::gc::StoreBuffer::traceWholeCells(TenuringTracer& mover) +{ + for (ArenaCellSet* cells = bufferWholeCell; cells; cells = cells->next) { + Arena* arena = cells->arena; + + MOZ_ASSERT(arena->bufferedCells == cells); + arena->bufferedCells = &ArenaCellSet::Empty; + + JS::TraceKind kind = MapAllocToTraceKind(arena->getAllocKind()); + switch (kind) { + case JS::TraceKind::Object: + TraceBufferedCells<JSObject>(mover, arena, cells); + break; + case JS::TraceKind::Script: + TraceBufferedCells<JSScript>(mover, arena, cells); + break; + case JS::TraceKind::JitCode: + TraceBufferedCells<jit::JitCode>(mover, arena, cells); + break; + default: + MOZ_CRASH("Unexpected trace kind"); + } + } + + bufferWholeCell = nullptr; +} + +void +js::gc::StoreBuffer::CellPtrEdge::trace(TenuringTracer& mover) const +{ + if (!*edge) + return; + + MOZ_ASSERT((*edge)->getTraceKind() == JS::TraceKind::Object); + mover.traverse(reinterpret_cast<JSObject**>(edge)); +} + +void +js::gc::StoreBuffer::ValueEdge::trace(TenuringTracer& mover) const +{ + if (deref()) + mover.traverse(edge); +} + +/* Insert the given relocation entry into the list of things to visit. */ +void +js::TenuringTracer::insertIntoFixupList(RelocationOverlay* entry) { + *tail = entry; + tail = &entry->nextRef(); + *tail = nullptr; +} + +JSObject* +js::TenuringTracer::moveToTenured(JSObject* src) +{ + MOZ_ASSERT(IsInsideNursery(src)); + MOZ_ASSERT(!src->zone()->usedByExclusiveThread); + + AllocKind dstKind = src->allocKindForTenure(nursery()); + Zone* zone = src->zone(); + + TenuredCell* t = zone->arenas.allocateFromFreeList(dstKind, Arena::thingSize(dstKind)); + if (!t) { + AutoEnterOOMUnsafeRegion oomUnsafe; + t = runtime()->gc.refillFreeListInGC(zone, dstKind); + if (!t) + oomUnsafe.crash(ChunkSize, "Failed to allocate object while tenuring."); + } + JSObject* dst = reinterpret_cast<JSObject*>(t); + tenuredSize += moveObjectToTenured(dst, src, dstKind); + + RelocationOverlay* overlay = RelocationOverlay::fromCell(src); + overlay->forwardTo(dst); + insertIntoFixupList(overlay); + + TracePromoteToTenured(src, dst); + MemProfiler::MoveNurseryToTenured(src, dst); + return dst; +} + +void +js::Nursery::collectToFixedPoint(TenuringTracer& mover, TenureCountCache& tenureCounts) +{ + for (RelocationOverlay* p = mover.head; p; p = p->next()) { + JSObject* obj = static_cast<JSObject*>(p->forwardingAddress()); + mover.traceObject(obj); + + TenureCount& entry = tenureCounts.findEntry(obj->groupRaw()); + if (entry.group == obj->groupRaw()) { + entry.count++; + } else if (!entry.group) { + entry.group = obj->groupRaw(); + entry.count = 1; + } + } +} + +struct TenuringFunctor +{ + template <typename T> + void operator()(T* thing, TenuringTracer& mover) { + mover.traverse(thing); + } +}; + +// Visit all object children of the object and trace them. +void +js::TenuringTracer::traceObject(JSObject* obj) +{ + NativeObject *nobj = CallTraceHook(TenuringFunctor(), this, obj, + CheckGeneration::NoChecks, *this); + if (!nobj) + return; + + // Note: the contents of copy on write elements pointers are filled in + // during parsing and cannot contain nursery pointers. + if (!nobj->hasEmptyElements() && + !nobj->denseElementsAreCopyOnWrite() && + ObjectDenseElementsMayBeMarkable(nobj)) + { + Value* elems = static_cast<HeapSlot*>(nobj->getDenseElements())->unsafeUnbarrieredForTracing(); + traceSlots(elems, elems + nobj->getDenseInitializedLength()); + } + + traceObjectSlots(nobj, 0, nobj->slotSpan()); +} + +void +js::TenuringTracer::traceObjectSlots(NativeObject* nobj, uint32_t start, uint32_t length) +{ + HeapSlot* fixedStart; + HeapSlot* fixedEnd; + HeapSlot* dynStart; + HeapSlot* dynEnd; + nobj->getSlotRange(start, length, &fixedStart, &fixedEnd, &dynStart, &dynEnd); + if (fixedStart) + traceSlots(fixedStart->unsafeUnbarrieredForTracing(), fixedEnd->unsafeUnbarrieredForTracing()); + if (dynStart) + traceSlots(dynStart->unsafeUnbarrieredForTracing(), dynEnd->unsafeUnbarrieredForTracing()); +} + +void +js::TenuringTracer::traceSlots(Value* vp, Value* end) +{ + for (; vp != end; ++vp) + traverse(vp); +} + +#ifdef DEBUG +static inline ptrdiff_t +OffsetToChunkEnd(void* p) +{ + return ChunkLocationOffset - (uintptr_t(p) & gc::ChunkMask); +} +#endif + +size_t +js::TenuringTracer::moveObjectToTenured(JSObject* dst, JSObject* src, AllocKind dstKind) +{ + size_t srcSize = Arena::thingSize(dstKind); + size_t tenuredSize = srcSize; + + /* + * Arrays do not necessarily have the same AllocKind between src and dst. + * We deal with this by copying elements manually, possibly re-inlining + * them if there is adequate room inline in dst. + * + * For Arrays we're reducing tenuredSize to the smaller srcSize + * because moveElementsToTenured() accounts for all Array elements, + * even if they are inlined. + */ + if (src->is<ArrayObject>()) { + tenuredSize = srcSize = sizeof(NativeObject); + } else if (src->is<TypedArrayObject>()) { + TypedArrayObject* tarray = &src->as<TypedArrayObject>(); + // Typed arrays with inline data do not necessarily have the same + // AllocKind between src and dst. The nursery does not allocate an + // inline data buffer that has the same size as the slow path will do. + // In the slow path, the Typed Array Object stores the inline data + // in the allocated space that fits the AllocKind. In the fast path, + // the nursery will allocate another buffer that is directly behind the + // minimal JSObject. That buffer size plus the JSObject size is not + // necessarily as large as the slow path's AllocKind size. + if (tarray->hasInlineElements()) { + AllocKind srcKind = GetGCObjectKind(TypedArrayObject::FIXED_DATA_START); + size_t headerSize = Arena::thingSize(srcKind); + srcSize = headerSize + tarray->byteLength(); + } + } + + // Copy the Cell contents. + MOZ_ASSERT(OffsetToChunkEnd(src) >= ptrdiff_t(srcSize)); + js_memcpy(dst, src, srcSize); + + // Move any hash code attached to the object. + src->zone()->transferUniqueId(dst, src); + + // Move the slots and elements, if we need to. + if (src->isNative()) { + NativeObject* ndst = &dst->as<NativeObject>(); + NativeObject* nsrc = &src->as<NativeObject>(); + tenuredSize += moveSlotsToTenured(ndst, nsrc, dstKind); + tenuredSize += moveElementsToTenured(ndst, nsrc, dstKind); + + // The shape's list head may point into the old object. This can only + // happen for dictionaries, which are native objects. + if (&nsrc->shape_ == ndst->shape_->listp) { + MOZ_ASSERT(nsrc->shape_->inDictionary()); + ndst->shape_->listp = &ndst->shape_; + } + } + + if (src->is<InlineTypedObject>()) { + InlineTypedObject::objectMovedDuringMinorGC(this, dst, src); + } else if (src->is<TypedArrayObject>()) { + tenuredSize += TypedArrayObject::objectMovedDuringMinorGC(this, dst, src, dstKind); + } else if (src->is<UnboxedArrayObject>()) { + tenuredSize += UnboxedArrayObject::objectMovedDuringMinorGC(this, dst, src, dstKind); + } else if (src->is<ArgumentsObject>()) { + tenuredSize += ArgumentsObject::objectMovedDuringMinorGC(this, dst, src); + } else if (src->is<ProxyObject>()) { + tenuredSize += ProxyObject::objectMovedDuringMinorGC(this, dst, src); + } else if (JSObjectMovedOp op = dst->getClass()->extObjectMovedOp()) { + op(dst, src); + } else if (src->getClass()->hasFinalize()) { + // Such objects need to be handled specially above to ensure any + // additional nursery buffers they hold are moved. + MOZ_RELEASE_ASSERT(CanNurseryAllocateFinalizedClass(src->getClass())); + MOZ_CRASH("Unhandled JSCLASS_SKIP_NURSERY_FINALIZE Class"); + } + + return tenuredSize; +} + +size_t +js::TenuringTracer::moveSlotsToTenured(NativeObject* dst, NativeObject* src, AllocKind dstKind) +{ + /* Fixed slots have already been copied over. */ + if (!src->hasDynamicSlots()) + return 0; + + if (!nursery().isInside(src->slots_)) { + nursery().removeMallocedBuffer(src->slots_); + return 0; + } + + Zone* zone = src->zone(); + size_t count = src->numDynamicSlots(); + + { + AutoEnterOOMUnsafeRegion oomUnsafe; + dst->slots_ = zone->pod_malloc<HeapSlot>(count); + if (!dst->slots_) + oomUnsafe.crash(sizeof(HeapSlot) * count, "Failed to allocate slots while tenuring."); + } + + PodCopy(dst->slots_, src->slots_, count); + nursery().setSlotsForwardingPointer(src->slots_, dst->slots_, count); + return count * sizeof(HeapSlot); +} + +size_t +js::TenuringTracer::moveElementsToTenured(NativeObject* dst, NativeObject* src, AllocKind dstKind) +{ + if (src->hasEmptyElements() || src->denseElementsAreCopyOnWrite()) + return 0; + + Zone* zone = src->zone(); + ObjectElements* srcHeader = src->getElementsHeader(); + ObjectElements* dstHeader; + + /* TODO Bug 874151: Prefer to put element data inline if we have space. */ + if (!nursery().isInside(srcHeader)) { + MOZ_ASSERT(src->elements_ == dst->elements_); + nursery().removeMallocedBuffer(srcHeader); + return 0; + } + + size_t nslots = ObjectElements::VALUES_PER_HEADER + srcHeader->capacity; + + /* Unlike other objects, Arrays can have fixed elements. */ + if (src->is<ArrayObject>() && nslots <= GetGCKindSlots(dstKind)) { + dst->as<ArrayObject>().setFixedElements(); + dstHeader = dst->as<ArrayObject>().getElementsHeader(); + js_memcpy(dstHeader, srcHeader, nslots * sizeof(HeapSlot)); + nursery().setElementsForwardingPointer(srcHeader, dstHeader, nslots); + return nslots * sizeof(HeapSlot); + } + + MOZ_ASSERT(nslots >= 2); + + { + AutoEnterOOMUnsafeRegion oomUnsafe; + dstHeader = reinterpret_cast<ObjectElements*>(zone->pod_malloc<HeapSlot>(nslots)); + if (!dstHeader) { + oomUnsafe.crash(sizeof(HeapSlot) * nslots, + "Failed to allocate elements while tenuring."); + } + } + + js_memcpy(dstHeader, srcHeader, nslots * sizeof(HeapSlot)); + nursery().setElementsForwardingPointer(srcHeader, dstHeader, nslots); + dst->elements_ = dstHeader->elements(); + return nslots * sizeof(HeapSlot); +} + + +/*** IsMarked / IsAboutToBeFinalized **************************************************************/ + +template <typename T> +static inline void +CheckIsMarkedThing(T* thingp) +{ +#define IS_SAME_TYPE_OR(name, type, _) mozilla::IsSame<type*, T>::value || + static_assert( + JS_FOR_EACH_TRACEKIND(IS_SAME_TYPE_OR) + false, "Only the base cell layout types are allowed into marking/tracing internals"); +#undef IS_SAME_TYPE_OR + +#ifdef DEBUG + MOZ_ASSERT(thingp); + MOZ_ASSERT(*thingp); + JSRuntime* rt = (*thingp)->runtimeFromAnyThread(); + MOZ_ASSERT_IF(!ThingIsPermanentAtomOrWellKnownSymbol(*thingp), + CurrentThreadCanAccessRuntime(rt) || + (rt->isHeapCollecting() && rt->gc.state() == State::Sweep)); +#endif +} + +template <typename T> +static bool +IsMarkedInternalCommon(T* thingp) +{ + CheckIsMarkedThing(thingp); + MOZ_ASSERT(!IsInsideNursery(*thingp)); + + Zone* zone = (*thingp)->asTenured().zoneFromAnyThread(); + if (!zone->isCollectingFromAnyThread() || zone->isGCFinished()) + return true; + if (zone->isGCCompacting() && IsForwarded(*thingp)) + *thingp = Forwarded(*thingp); + return (*thingp)->asTenured().isMarked(); +} + +template <typename T> +static bool +IsMarkedInternal(JSRuntime* rt, T** thingp) +{ + if (IsOwnedByOtherRuntime(rt, *thingp)) + return true; + + return IsMarkedInternalCommon(thingp); +} + +template <> +/* static */ bool +IsMarkedInternal(JSRuntime* rt, JSObject** thingp) +{ + if (IsOwnedByOtherRuntime(rt, *thingp)) + return true; + + if (IsInsideNursery(*thingp)) { + MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt)); + return rt->gc.nursery.getForwardedPointer(thingp); + } + return IsMarkedInternalCommon(thingp); +} + +template <typename S> +struct IsMarkedFunctor : public IdentityDefaultAdaptor<S> { + template <typename T> S operator()(T* t, JSRuntime* rt, bool* rv) { + *rv = IsMarkedInternal(rt, &t); + return js::gc::RewrapTaggedPointer<S, T>::wrap(t); + } +}; + +template <typename T> +static bool +IsMarkedInternal(JSRuntime* rt, T* thingp) +{ + bool rv = true; + *thingp = DispatchTyped(IsMarkedFunctor<T>(), *thingp, rt, &rv); + return rv; +} + +bool +js::gc::IsAboutToBeFinalizedDuringSweep(TenuredCell& tenured) +{ + MOZ_ASSERT(!IsInsideNursery(&tenured)); + MOZ_ASSERT(tenured.zoneFromAnyThread()->isGCSweeping()); + if (tenured.arena()->allocatedDuringIncremental) + return false; + return !tenured.isMarked(); +} + +template <typename T> +static bool +IsAboutToBeFinalizedInternal(T** thingp) +{ + CheckIsMarkedThing(thingp); + T* thing = *thingp; + JSRuntime* rt = thing->runtimeFromAnyThread(); + + /* Permanent atoms are never finalized by non-owning runtimes. */ + if (ThingIsPermanentAtomOrWellKnownSymbol(thing) && !TlsPerThreadData.get()->associatedWith(rt)) + return false; + + Nursery& nursery = rt->gc.nursery; + if (IsInsideNursery(thing)) { + MOZ_ASSERT(rt->isHeapMinorCollecting()); + return !nursery.getForwardedPointer(reinterpret_cast<JSObject**>(thingp)); + } + + Zone* zone = thing->asTenured().zoneFromAnyThread(); + if (zone->isGCSweeping()) { + return IsAboutToBeFinalizedDuringSweep(thing->asTenured()); + } else if (zone->isGCCompacting() && IsForwarded(thing)) { + *thingp = Forwarded(thing); + return false; + } + + return false; +} + +template <typename S> +struct IsAboutToBeFinalizedFunctor : public IdentityDefaultAdaptor<S> { + template <typename T> S operator()(T* t, bool* rv) { + *rv = IsAboutToBeFinalizedInternal(&t); + return js::gc::RewrapTaggedPointer<S, T>::wrap(t); + } +}; + +template <typename T> +static bool +IsAboutToBeFinalizedInternal(T* thingp) +{ + bool rv = false; + *thingp = DispatchTyped(IsAboutToBeFinalizedFunctor<T>(), *thingp, &rv); + return rv; +} + +namespace js { +namespace gc { + +template <typename T> +bool +IsMarkedUnbarriered(JSRuntime* rt, T* thingp) +{ + return IsMarkedInternal(rt, ConvertToBase(thingp)); +} + +template <typename T> +bool +IsMarked(JSRuntime* rt, WriteBarrieredBase<T>* thingp) +{ + return IsMarkedInternal(rt, ConvertToBase(thingp->unsafeUnbarrieredForTracing())); +} + +template <typename T> +bool +IsAboutToBeFinalizedUnbarriered(T* thingp) +{ + return IsAboutToBeFinalizedInternal(ConvertToBase(thingp)); +} + +template <typename T> +bool +IsAboutToBeFinalized(WriteBarrieredBase<T>* thingp) +{ + return IsAboutToBeFinalizedInternal(ConvertToBase(thingp->unsafeUnbarrieredForTracing())); +} + +template <typename T> +bool +IsAboutToBeFinalized(ReadBarrieredBase<T>* thingp) +{ + return IsAboutToBeFinalizedInternal(ConvertToBase(thingp->unsafeUnbarrieredForTracing())); +} + +template <typename T> +JS_PUBLIC_API(bool) +EdgeNeedsSweep(JS::Heap<T>* thingp) +{ + return IsAboutToBeFinalizedInternal(ConvertToBase(thingp->unsafeGet())); +} + +// Instantiate a copy of the Tracing templates for each derived type. +#define INSTANTIATE_ALL_VALID_TRACE_FUNCTIONS(type) \ + template bool IsMarkedUnbarriered<type>(JSRuntime*, type*); \ + template bool IsMarked<type>(JSRuntime*, WriteBarrieredBase<type>*); \ + template bool IsAboutToBeFinalizedUnbarriered<type>(type*); \ + template bool IsAboutToBeFinalized<type>(WriteBarrieredBase<type>*); \ + template bool IsAboutToBeFinalized<type>(ReadBarrieredBase<type>*); +#define INSTANTIATE_ALL_VALID_HEAP_TRACE_FUNCTIONS(type) \ + template JS_PUBLIC_API(bool) EdgeNeedsSweep<type>(JS::Heap<type>*); +FOR_EACH_GC_POINTER_TYPE(INSTANTIATE_ALL_VALID_TRACE_FUNCTIONS) +FOR_EACH_PUBLIC_GC_POINTER_TYPE(INSTANTIATE_ALL_VALID_HEAP_TRACE_FUNCTIONS) +FOR_EACH_PUBLIC_TAGGED_GC_POINTER_TYPE(INSTANTIATE_ALL_VALID_HEAP_TRACE_FUNCTIONS) +#undef INSTANTIATE_ALL_VALID_TRACE_FUNCTIONS + +} /* namespace gc */ +} /* namespace js */ + + +/*** Cycle Collector Barrier Implementation *******************************************************/ + +#ifdef DEBUG +struct AssertNonGrayTracer : public JS::CallbackTracer { + explicit AssertNonGrayTracer(JSRuntime* rt) : JS::CallbackTracer(rt) {} + void onChild(const JS::GCCellPtr& thing) override { + MOZ_ASSERT_IF(thing.asCell()->isTenured(), + !thing.asCell()->asTenured().isMarked(js::gc::GRAY)); + } +}; +#endif + +struct UnmarkGrayTracer : public JS::CallbackTracer +{ + /* + * We set weakMapAction to DoNotTraceWeakMaps because the cycle collector + * will fix up any color mismatches involving weakmaps when it runs. + */ + explicit UnmarkGrayTracer(JSRuntime *rt, bool tracingShape = false) + : JS::CallbackTracer(rt, DoNotTraceWeakMaps) + , tracingShape(tracingShape) + , previousShape(nullptr) + , unmarkedAny(false) + {} + + void onChild(const JS::GCCellPtr& thing) override; + + /* True iff we are tracing the immediate children of a shape. */ + bool tracingShape; + + /* If tracingShape, shape child or nullptr. Otherwise, nullptr. */ + Shape* previousShape; + + /* Whether we unmarked anything. */ + bool unmarkedAny; +}; + +/* + * The GC and CC are run independently. Consequently, the following sequence of + * events can occur: + * 1. GC runs and marks an object gray. + * 2. The mutator runs (specifically, some C++ code with access to gray + * objects) and creates a pointer from a JS root or other black object to + * the gray object. If we re-ran a GC at this point, the object would now be + * black. + * 3. Now we run the CC. It may think it can collect the gray object, even + * though it's reachable from the JS heap. + * + * To prevent this badness, we unmark the gray bit of an object when it is + * accessed by callers outside XPConnect. This would cause the object to go + * black in step 2 above. This must be done on everything reachable from the + * object being returned. The following code takes care of the recursive + * re-coloring. + * + * There is an additional complication for certain kinds of edges that are not + * contained explicitly in the source object itself, such as from a weakmap key + * to its value, and from an object being watched by a watchpoint to the + * watchpoint's closure. These "implicit edges" are represented in some other + * container object, such as the weakmap or the watchpoint itself. In these + * cases, calling unmark gray on an object won't find all of its children. + * + * Handling these implicit edges has two parts: + * - A special pass enumerating all of the containers that know about the + * implicit edges to fix any black-gray edges that have been created. This + * is implemented in nsXPConnect::FixWeakMappingGrayBits. + * - To prevent any incorrectly gray objects from escaping to live JS outside + * of the containers, we must add unmark-graying read barriers to these + * containers. + */ +void +UnmarkGrayTracer::onChild(const JS::GCCellPtr& thing) +{ + int stackDummy; + JSContext* cx = runtime()->contextFromMainThread(); + if (!JS_CHECK_STACK_SIZE(cx->nativeStackLimit[StackForSystemCode], &stackDummy)) { + /* + * If we run out of stack, we take a more drastic measure: require that + * we GC again before the next CC. + */ + runtime()->setGCGrayBitsValid(false); + return; + } + + Cell* cell = thing.asCell(); + + // Cells in the nursery cannot be gray, and therefore must necessarily point + // to only black edges. + if (!cell->isTenured()) { +#ifdef DEBUG + AssertNonGrayTracer nongray(runtime()); + TraceChildren(&nongray, cell, thing.kind()); +#endif + return; + } + + TenuredCell& tenured = cell->asTenured(); + if (!tenured.isMarked(js::gc::GRAY)) + return; + tenured.unmark(js::gc::GRAY); + + unmarkedAny = true; + + // Trace children of |tenured|. If |tenured| and its parent are both + // shapes, |tenured| will get saved to mPreviousShape without being traced. + // The parent will later trace |tenured|. This is done to avoid increasing + // the stack depth during shape tracing. It is safe to do because a shape + // can only have one child that is a shape. + UnmarkGrayTracer childTracer(runtime(), thing.kind() == JS::TraceKind::Shape); + + if (thing.kind() != JS::TraceKind::Shape) { + TraceChildren(&childTracer, &tenured, thing.kind()); + MOZ_ASSERT(!childTracer.previousShape); + unmarkedAny |= childTracer.unmarkedAny; + return; + } + + MOZ_ASSERT(thing.kind() == JS::TraceKind::Shape); + Shape* shape = static_cast<Shape*>(&tenured); + if (tracingShape) { + MOZ_ASSERT(!previousShape); + previousShape = shape; + return; + } + + do { + MOZ_ASSERT(!shape->isMarked(js::gc::GRAY)); + shape->traceChildren(&childTracer); + shape = childTracer.previousShape; + childTracer.previousShape = nullptr; + } while (shape); + unmarkedAny |= childTracer.unmarkedAny; +} + +template <typename T> +static bool +TypedUnmarkGrayCellRecursively(T* t) +{ + MOZ_ASSERT(t); + + JSRuntime* rt = t->runtimeFromMainThread(); + MOZ_ASSERT(!rt->isHeapCollecting()); + MOZ_ASSERT(!rt->isCycleCollecting()); + + bool unmarkedArg = false; + if (t->isTenured()) { + if (!t->asTenured().isMarked(GRAY)) + return false; + + t->asTenured().unmark(GRAY); + unmarkedArg = true; + } + + UnmarkGrayTracer trc(rt); + gcstats::AutoPhase outerPhase(rt->gc.stats, gcstats::PHASE_BARRIER); + gcstats::AutoPhase innerPhase(rt->gc.stats, gcstats::PHASE_UNMARK_GRAY); + t->traceChildren(&trc); + + return unmarkedArg || trc.unmarkedAny; +} + +struct UnmarkGrayCellRecursivelyFunctor { + template <typename T> bool operator()(T* t) { return TypedUnmarkGrayCellRecursively(t); } +}; + +bool +js::UnmarkGrayCellRecursively(Cell* cell, JS::TraceKind kind) +{ + return DispatchTraceKindTyped(UnmarkGrayCellRecursivelyFunctor(), cell, kind); +} + +bool +js::UnmarkGrayShapeRecursively(Shape* shape) +{ + return TypedUnmarkGrayCellRecursively(shape); +} + +JS_FRIEND_API(bool) +JS::UnmarkGrayGCThingRecursively(JS::GCCellPtr thing) +{ + return js::UnmarkGrayCellRecursively(thing.asCell(), thing.kind()); +} diff --git a/js/src/gc/Marking.h b/js/src/gc/Marking.h new file mode 100644 index 000000000..ec4c69a2f --- /dev/null +++ b/js/src/gc/Marking.h @@ -0,0 +1,477 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_Marking_h +#define gc_Marking_h + +#include "mozilla/HashFunctions.h" +#include "mozilla/Move.h" + +#include "jsfriendapi.h" + +#include "ds/OrderedHashTable.h" +#include "gc/Heap.h" +#include "gc/Tracer.h" +#include "js/GCAPI.h" +#include "js/HeapAPI.h" +#include "js/SliceBudget.h" +#include "js/TracingAPI.h" +#include "vm/TaggedProto.h" + +class JSLinearString; +class JSRope; +namespace js { +class BaseShape; +class GCMarker; +class LazyScript; +class NativeObject; +class ObjectGroup; +class WeakMapBase; +namespace gc { +class Arena; +} // namespace gc +namespace jit { +class JitCode; +} // namespace jit + +static const size_t NON_INCREMENTAL_MARK_STACK_BASE_CAPACITY = 4096; +static const size_t INCREMENTAL_MARK_STACK_BASE_CAPACITY = 32768; + +/* + * When the native stack is low, the GC does not call js::TraceChildren to mark + * the reachable "children" of the thing. Rather the thing is put aside and + * js::TraceChildren is called later with more space on the C stack. + * + * To implement such delayed marking of the children with minimal overhead for + * the normal case of sufficient native stack, the code adds a field per arena. + * The field markingDelay->link links all arenas with delayed things into a + * stack list with the pointer to stack top in GCMarker::unmarkedArenaStackTop. + * GCMarker::delayMarkingChildren adds arenas to the stack as necessary while + * markDelayedChildren pops the arenas from the stack until it empties. + */ +class MarkStack +{ + friend class GCMarker; + + uintptr_t* stack_; + uintptr_t* tos_; + uintptr_t* end_; + + // The capacity we start with and reset() to. + size_t baseCapacity_; + size_t maxCapacity_; + + public: + explicit MarkStack(size_t maxCapacity) + : stack_(nullptr), + tos_(nullptr), + end_(nullptr), + baseCapacity_(0), + maxCapacity_(maxCapacity) + {} + + ~MarkStack() { + js_free(stack_); + } + + size_t capacity() { return end_ - stack_; } + + ptrdiff_t position() const { return tos_ - stack_; } + + void setStack(uintptr_t* stack, size_t tosIndex, size_t capacity) { + stack_ = stack; + tos_ = stack + tosIndex; + end_ = stack + capacity; + } + + MOZ_MUST_USE bool init(JSGCMode gcMode); + + void setBaseCapacity(JSGCMode mode); + size_t maxCapacity() const { return maxCapacity_; } + void setMaxCapacity(size_t maxCapacity); + + MOZ_MUST_USE bool push(uintptr_t item) { + if (tos_ == end_) { + if (!enlarge(1)) + return false; + } + MOZ_ASSERT(tos_ < end_); + *tos_++ = item; + return true; + } + + MOZ_MUST_USE bool push(uintptr_t item1, uintptr_t item2, uintptr_t item3) { + uintptr_t* nextTos = tos_ + 3; + if (nextTos > end_) { + if (!enlarge(3)) + return false; + nextTos = tos_ + 3; + } + MOZ_ASSERT(nextTos <= end_); + tos_[0] = item1; + tos_[1] = item2; + tos_[2] = item3; + tos_ = nextTos; + return true; + } + + bool isEmpty() const { + return tos_ == stack_; + } + + uintptr_t pop() { + MOZ_ASSERT(!isEmpty()); + return *--tos_; + } + + void reset(); + + /* Grow the stack, ensuring there is space for at least count elements. */ + MOZ_MUST_USE bool enlarge(unsigned count); + + void setGCMode(JSGCMode gcMode); + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; +}; + +namespace gc { + +struct WeakKeyTableHashPolicy { + typedef JS::GCCellPtr Lookup; + static HashNumber hash(const Lookup& v, const mozilla::HashCodeScrambler&) { + return mozilla::HashGeneric(v.asCell()); + } + static bool match(const JS::GCCellPtr& k, const Lookup& l) { return k == l; } + static bool isEmpty(const JS::GCCellPtr& v) { return !v; } + static void makeEmpty(JS::GCCellPtr* vp) { *vp = nullptr; } +}; + +struct WeakMarkable { + WeakMapBase* weakmap; + JS::GCCellPtr key; + + WeakMarkable(WeakMapBase* weakmapArg, JS::GCCellPtr keyArg) + : weakmap(weakmapArg), key(keyArg) {} +}; + +using WeakEntryVector = Vector<WeakMarkable, 2, js::SystemAllocPolicy>; + +using WeakKeyTable = OrderedHashMap<JS::GCCellPtr, + WeakEntryVector, + WeakKeyTableHashPolicy, + js::SystemAllocPolicy>; + +} /* namespace gc */ + +class GCMarker : public JSTracer +{ + public: + explicit GCMarker(JSRuntime* rt); + MOZ_MUST_USE bool init(JSGCMode gcMode); + + void setMaxCapacity(size_t maxCap) { stack.setMaxCapacity(maxCap); } + size_t maxCapacity() const { return stack.maxCapacity(); } + + void start(); + void stop(); + void reset(); + + // Mark the given GC thing and traverse its children at some point. + template <typename T> void traverse(T thing); + + // Calls traverse on target after making additional assertions. + template <typename S, typename T> void traverseEdge(S source, T* target); + template <typename S, typename T> void traverseEdge(S source, const T& target); + + // Notes a weak graph edge for later sweeping. + template <typename T> void noteWeakEdge(T* edge); + + /* + * Care must be taken changing the mark color from gray to black. The cycle + * collector depends on the invariant that there are no black to gray edges + * in the GC heap. This invariant lets the CC not trace through black + * objects. If this invariant is violated, the cycle collector may free + * objects that are still reachable. + */ + void setMarkColorGray() { + MOZ_ASSERT(isDrained()); + MOZ_ASSERT(color == gc::BLACK); + color = gc::GRAY; + } + void setMarkColorBlack() { + MOZ_ASSERT(isDrained()); + MOZ_ASSERT(color == gc::GRAY); + color = gc::BLACK; + } + uint32_t markColor() const { return color; } + + void enterWeakMarkingMode(); + void leaveWeakMarkingMode(); + void abortLinearWeakMarking() { + leaveWeakMarkingMode(); + linearWeakMarkingDisabled_ = true; + } + + void delayMarkingArena(gc::Arena* arena); + void delayMarkingChildren(const void* thing); + void markDelayedChildren(gc::Arena* arena); + MOZ_MUST_USE bool markDelayedChildren(SliceBudget& budget); + bool hasDelayedChildren() const { + return !!unmarkedArenaStackTop; + } + + bool isDrained() { + return isMarkStackEmpty() && !unmarkedArenaStackTop; + } + + MOZ_MUST_USE bool drainMarkStack(SliceBudget& budget); + + void setGCMode(JSGCMode mode) { stack.setGCMode(mode); } + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const; + +#ifdef DEBUG + bool shouldCheckCompartments() { return strictCompartmentChecking; } +#endif + + void markEphemeronValues(gc::Cell* markedCell, gc::WeakEntryVector& entry); + + private: +#ifdef DEBUG + void checkZone(void* p); +#else + void checkZone(void* p) {} +#endif + + /* + * We use a common mark stack to mark GC things of different types and use + * the explicit tags to distinguish them when it cannot be deduced from + * the context of push or pop operation. + */ + enum StackTag { + ValueArrayTag, + ObjectTag, + GroupTag, + SavedValueArrayTag, + JitCodeTag, + ScriptTag, + LastTag = JitCodeTag + }; + + static const uintptr_t StackTagMask = 7; + static_assert(StackTagMask >= uintptr_t(LastTag), "The tag mask must subsume the tags."); + static_assert(StackTagMask <= gc::CellMask, "The tag mask must be embeddable in a Cell*."); + + // Push an object onto the stack for later tracing and assert that it has + // already been marked. + void repush(JSObject* obj) { + MOZ_ASSERT(gc::TenuredCell::fromPointer(obj)->isMarked(markColor())); + pushTaggedPtr(ObjectTag, obj); + } + + template <typename T> void markAndTraceChildren(T* thing); + template <typename T> void markAndPush(StackTag tag, T* thing); + template <typename T> void markAndScan(T* thing); + template <typename T> void markImplicitEdgesHelper(T oldThing); + template <typename T> void markImplicitEdges(T* oldThing); + void eagerlyMarkChildren(JSLinearString* str); + void eagerlyMarkChildren(JSRope* rope); + void eagerlyMarkChildren(JSString* str); + void eagerlyMarkChildren(LazyScript *thing); + void eagerlyMarkChildren(Shape* shape); + void eagerlyMarkChildren(Scope* scope); + void lazilyMarkChildren(ObjectGroup* group); + + // We may not have concrete types yet, so this has to be outside the header. + template <typename T> + void dispatchToTraceChildren(T* thing); + + // Mark the given GC thing, but do not trace its children. Return true + // if the thing became marked. + template <typename T> + MOZ_MUST_USE bool mark(T* thing); + + void pushTaggedPtr(StackTag tag, void* ptr) { + checkZone(ptr); + uintptr_t addr = reinterpret_cast<uintptr_t>(ptr); + MOZ_ASSERT(!(addr & StackTagMask)); + if (!stack.push(addr | uintptr_t(tag))) + delayMarkingChildren(ptr); + } + + void pushValueArray(JSObject* obj, HeapSlot* start, HeapSlot* end) { + checkZone(obj); + + MOZ_ASSERT(start <= end); + uintptr_t tagged = reinterpret_cast<uintptr_t>(obj) | GCMarker::ValueArrayTag; + uintptr_t startAddr = reinterpret_cast<uintptr_t>(start); + uintptr_t endAddr = reinterpret_cast<uintptr_t>(end); + + /* + * Push in the reverse order so obj will be on top. If we cannot push + * the array, we trigger delay marking for the whole object. + */ + if (!stack.push(endAddr, startAddr, tagged)) + delayMarkingChildren(obj); + } + + bool isMarkStackEmpty() { + return stack.isEmpty(); + } + + MOZ_MUST_USE bool restoreValueArray(JSObject* obj, void** vpp, void** endp); + void saveValueRanges(); + inline void processMarkStackTop(SliceBudget& budget); + + /* The mark stack. Pointers in this stack are "gray" in the GC sense. */ + MarkStack stack; + + /* The color is only applied to objects and functions. */ + uint32_t color; + + /* Pointer to the top of the stack of arenas we are delaying marking on. */ + js::gc::Arena* unmarkedArenaStackTop; + + /* + * If the weakKeys table OOMs, disable the linear algorithm and fall back + * to iterating until the next GC. + */ + bool linearWeakMarkingDisabled_; + +#ifdef DEBUG + /* Count of arenas that are currently in the stack. */ + size_t markLaterArenas; + + /* Assert that start and stop are called with correct ordering. */ + bool started; + + /* + * If this is true, all marked objects must belong to a compartment being + * GCed. This is used to look for compartment bugs. + */ + bool strictCompartmentChecking; +#endif // DEBUG +}; + +#ifdef DEBUG +// Return true if this trace is happening on behalf of gray buffering during +// the marking phase of incremental GC. +bool +IsBufferGrayRootsTracer(JSTracer* trc); +#endif + +namespace gc { + +/*** Special Cases ***/ + +void +PushArena(GCMarker* gcmarker, Arena* arena); + +/*** Liveness ***/ + +// Report whether a thing has been marked. Things which are in zones that are +// not currently being collected or are owned by another runtime are always +// reported as being marked. +template <typename T> +bool +IsMarkedUnbarriered(JSRuntime* rt, T* thingp); + +// Report whether a thing has been marked. Things which are in zones that are +// not currently being collected or are owned by another runtime are always +// reported as being marked. +template <typename T> +bool +IsMarked(JSRuntime* rt, WriteBarrieredBase<T>* thingp); + +template <typename T> +bool +IsAboutToBeFinalizedUnbarriered(T* thingp); + +template <typename T> +bool +IsAboutToBeFinalized(WriteBarrieredBase<T>* thingp); + +template <typename T> +bool +IsAboutToBeFinalized(ReadBarrieredBase<T>* thingp); + +bool +IsAboutToBeFinalizedDuringSweep(TenuredCell& tenured); + +inline Cell* +ToMarkable(const Value& v) +{ + if (v.isMarkable()) + return (Cell*)v.toGCThing(); + return nullptr; +} + +inline Cell* +ToMarkable(Cell* cell) +{ + return cell; +} + +// Return true if the pointer is nullptr, or if it is a tagged pointer to +// nullptr. +MOZ_ALWAYS_INLINE bool +IsNullTaggedPointer(void* p) +{ + return uintptr_t(p) <= LargestTaggedNullCellPointer; +} + +// Wrap a GC thing pointer into a new Value or jsid. The type system enforces +// that the thing pointer is a wrappable type. +template <typename S, typename T> +struct RewrapTaggedPointer{}; +#define DECLARE_REWRAP(S, T, method, prefix) \ + template <> struct RewrapTaggedPointer<S, T> { \ + static S wrap(T* thing) { return method ( prefix thing ); } \ + } +DECLARE_REWRAP(JS::Value, JSObject, JS::ObjectOrNullValue, ); +DECLARE_REWRAP(JS::Value, JSString, JS::StringValue, ); +DECLARE_REWRAP(JS::Value, JS::Symbol, JS::SymbolValue, ); +DECLARE_REWRAP(jsid, JSString, NON_INTEGER_ATOM_TO_JSID, (JSAtom*)); +DECLARE_REWRAP(jsid, JS::Symbol, SYMBOL_TO_JSID, ); +DECLARE_REWRAP(js::TaggedProto, JSObject, js::TaggedProto, ); +#undef DECLARE_REWRAP + +template <typename T> +struct IsPrivateGCThingInValue + : public mozilla::EnableIf<mozilla::IsBaseOf<Cell, T>::value && + !mozilla::IsBaseOf<JSObject, T>::value && + !mozilla::IsBaseOf<JSString, T>::value && + !mozilla::IsBaseOf<JS::Symbol, T>::value, T> +{ + static_assert(!mozilla::IsSame<Cell, T>::value && !mozilla::IsSame<TenuredCell, T>::value, + "T must not be Cell or TenuredCell"); +}; + +template <typename T> +struct RewrapTaggedPointer<Value, T> +{ + static Value wrap(typename IsPrivateGCThingInValue<T>::Type* thing) { + return JS::PrivateGCThingValue(thing); + } +}; + +} /* namespace gc */ + +// The return value indicates if anything was unmarked. +bool +UnmarkGrayShapeRecursively(Shape* shape); + +template<typename T> +void +CheckTracedThing(JSTracer* trc, T* thing); + +template<typename T> +void +CheckTracedThing(JSTracer* trc, T thing); + +} /* namespace js */ + +#endif /* gc_Marking_h */ diff --git a/js/src/gc/Memory.cpp b/js/src/gc/Memory.cpp new file mode 100644 index 000000000..26da75469 --- /dev/null +++ b/js/src/gc/Memory.cpp @@ -0,0 +1,901 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/Memory.h" + +#include "mozilla/Atomics.h" +#include "mozilla/TaggedAnonymousMemory.h" + +#include "js/HeapAPI.h" +#include "vm/Runtime.h" + +#if defined(XP_WIN) + +#include "jswin.h" +#include <psapi.h> + +#elif defined(SOLARIS) + +#include <sys/mman.h> +#include <unistd.h> + +#elif defined(XP_UNIX) + +#include <algorithm> +#include <errno.h> +#include <sys/mman.h> +#include <sys/resource.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#endif + +namespace js { +namespace gc { + +// The GC can only safely decommit memory when the page size of the +// running process matches the compiled arena size. +static size_t pageSize = 0; + +// The OS allocation granularity may not match the page size. +static size_t allocGranularity = 0; + +#if defined(XP_UNIX) +// The addresses handed out by mmap may grow up or down. +static mozilla::Atomic<int, mozilla::Relaxed> growthDirection(0); +#endif + +// Data from OOM crashes shows there may be up to 24 chunksized but unusable +// chunks available in low memory situations. These chunks may all need to be +// used up before we gain access to remaining *alignable* chunksized regions, +// so we use a generous limit of 32 unusable chunks to ensure we reach them. +static const int MaxLastDitchAttempts = 32; + +static void GetNewChunk(void** aAddress, void** aRetainedAddr, size_t size, size_t alignment); +static void* MapAlignedPagesSlow(size_t size, size_t alignment); +static void* MapAlignedPagesLastDitch(size_t size, size_t alignment); + +size_t +SystemPageSize() +{ + return pageSize; +} + +static bool +DecommitEnabled() +{ + return pageSize == ArenaSize; +} + +/* + * This returns the offset of address p from the nearest aligned address at + * or below p - or alternatively, the number of unaligned bytes at the end of + * the region starting at p (as we assert that allocation size is an integer + * multiple of the alignment). + */ +static inline size_t +OffsetFromAligned(void* p, size_t alignment) +{ + return uintptr_t(p) % alignment; +} + +void* +TestMapAlignedPagesLastDitch(size_t size, size_t alignment) +{ + return MapAlignedPagesLastDitch(size, alignment); +} + + +#if defined(XP_WIN) + +void +InitMemorySubsystem() +{ + if (pageSize == 0) { + SYSTEM_INFO sysinfo; + GetSystemInfo(&sysinfo); + pageSize = sysinfo.dwPageSize; + allocGranularity = sysinfo.dwAllocationGranularity; + } +} + +# if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) + +static inline void* +MapMemoryAt(void* desired, size_t length, int flags, int prot = PAGE_READWRITE) +{ + return VirtualAlloc(desired, length, flags, prot); +} + +static inline void* +MapMemory(size_t length, int flags, int prot = PAGE_READWRITE) +{ + return VirtualAlloc(nullptr, length, flags, prot); +} + +void* +MapAlignedPages(size_t size, size_t alignment) +{ + MOZ_ASSERT(size >= alignment); + MOZ_ASSERT(size >= allocGranularity); + MOZ_ASSERT(size % alignment == 0); + MOZ_ASSERT(size % pageSize == 0); + MOZ_ASSERT_IF(alignment < allocGranularity, allocGranularity % alignment == 0); + MOZ_ASSERT_IF(alignment > allocGranularity, alignment % allocGranularity == 0); + + void* p = MapMemory(size, MEM_COMMIT | MEM_RESERVE); + + /* Special case: If we want allocation alignment, no further work is needed. */ + if (alignment == allocGranularity) + return p; + + if (OffsetFromAligned(p, alignment) == 0) + return p; + + void* retainedAddr; + GetNewChunk(&p, &retainedAddr, size, alignment); + if (retainedAddr) + UnmapPages(retainedAddr, size); + if (p) { + if (OffsetFromAligned(p, alignment) == 0) + return p; + UnmapPages(p, size); + } + + p = MapAlignedPagesSlow(size, alignment); + if (!p) + return MapAlignedPagesLastDitch(size, alignment); + + MOZ_ASSERT(OffsetFromAligned(p, alignment) == 0); + return p; +} + +static void* +MapAlignedPagesSlow(size_t size, size_t alignment) +{ + /* + * Windows requires that there be a 1:1 mapping between VM allocation + * and deallocation operations. Therefore, take care here to acquire the + * final result via one mapping operation. This means unmapping any + * preliminary result that is not correctly aligned. + */ + void* p; + do { + /* + * Over-allocate in order to map a memory region that is definitely + * large enough, then deallocate and allocate again the correct size, + * within the over-sized mapping. + * + * Since we're going to unmap the whole thing anyway, the first + * mapping doesn't have to commit pages. + */ + size_t reserveSize = size + alignment - pageSize; + p = MapMemory(reserveSize, MEM_RESERVE); + if (!p) + return nullptr; + void* chunkStart = (void*)AlignBytes(uintptr_t(p), alignment); + UnmapPages(p, reserveSize); + p = MapMemoryAt(chunkStart, size, MEM_COMMIT | MEM_RESERVE); + + /* Failure here indicates a race with another thread, so try again. */ + } while (!p); + + return p; +} + +/* + * In a low memory or high fragmentation situation, alignable chunks of the + * desired size may still be available, even if there are no more contiguous + * free chunks that meet the |size + alignment - pageSize| requirement of + * MapAlignedPagesSlow. In this case, try harder to find an alignable chunk + * by temporarily holding onto the unaligned parts of each chunk until the + * allocator gives us a chunk that either is, or can be aligned. + */ +static void* +MapAlignedPagesLastDitch(size_t size, size_t alignment) +{ + void* tempMaps[MaxLastDitchAttempts]; + int attempt = 0; + void* p = MapMemory(size, MEM_COMMIT | MEM_RESERVE); + if (OffsetFromAligned(p, alignment) == 0) + return p; + for (; attempt < MaxLastDitchAttempts; ++attempt) { + GetNewChunk(&p, tempMaps + attempt, size, alignment); + if (OffsetFromAligned(p, alignment) == 0) { + if (tempMaps[attempt]) + UnmapPages(tempMaps[attempt], size); + break; + } + if (!tempMaps[attempt]) + break; /* Bail if GetNewChunk failed. */ + } + if (OffsetFromAligned(p, alignment)) { + UnmapPages(p, size); + p = nullptr; + } + while (--attempt >= 0) + UnmapPages(tempMaps[attempt], size); + return p; +} + +/* + * On Windows, map and unmap calls must be matched, so we deallocate the + * unaligned chunk, then reallocate the unaligned part to block off the + * old address and force the allocator to give us a new one. + */ +static void +GetNewChunk(void** aAddress, void** aRetainedAddr, size_t size, size_t alignment) +{ + void* address = *aAddress; + void* retainedAddr = nullptr; + do { + size_t retainedSize; + size_t offset = OffsetFromAligned(address, alignment); + if (!offset) + break; + UnmapPages(address, size); + retainedSize = alignment - offset; + retainedAddr = MapMemoryAt(address, retainedSize, MEM_RESERVE); + address = MapMemory(size, MEM_COMMIT | MEM_RESERVE); + /* If retainedAddr is null here, we raced with another thread. */ + } while (!retainedAddr); + *aAddress = address; + *aRetainedAddr = retainedAddr; +} + +void +UnmapPages(void* p, size_t size) +{ + MOZ_ALWAYS_TRUE(VirtualFree(p, 0, MEM_RELEASE)); +} + +bool +MarkPagesUnused(void* p, size_t size) +{ + if (!DecommitEnabled()) + return true; + + MOZ_ASSERT(OffsetFromAligned(p, pageSize) == 0); + LPVOID p2 = MapMemoryAt(p, size, MEM_RESET); + return p2 == p; +} + +void +MarkPagesInUse(void* p, size_t size) +{ + if (!DecommitEnabled()) + return; + + MOZ_ASSERT(OffsetFromAligned(p, pageSize) == 0); +} + +size_t +GetPageFaultCount() +{ + PROCESS_MEMORY_COUNTERS pmc; + if (!GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc))) + return 0; + return pmc.PageFaultCount; +} + +void* +AllocateMappedContent(int fd, size_t offset, size_t length, size_t alignment) +{ + MOZ_ASSERT(length && alignment); + + // The allocation granularity and the requested offset + // must both be divisible by the requested alignment. + // Alignments larger than the allocation granularity are not supported. + if (allocGranularity % alignment != 0 || offset % alignment != 0) + return nullptr; + + HANDLE hFile = reinterpret_cast<HANDLE>(intptr_t(fd)); + + // This call will fail if the file does not exist, which is what we want. + HANDLE hMap = CreateFileMapping(hFile, nullptr, PAGE_READONLY, 0, 0, nullptr); + if (!hMap) + return nullptr; + + size_t alignedOffset = offset - (offset % allocGranularity); + size_t alignedLength = length + (offset % allocGranularity); + + DWORD offsetH = uint32_t(uint64_t(alignedOffset) >> 32); + DWORD offsetL = uint32_t(alignedOffset); + + // If the offset or length are out of bounds, this call will fail. + uint8_t* map = static_cast<uint8_t*>(MapViewOfFile(hMap, FILE_MAP_COPY, offsetH, + offsetL, alignedLength)); + + // This just decreases the file mapping object's internal reference count; + // it won't actually be destroyed until we unmap the associated view. + CloseHandle(hMap); + + if (!map) + return nullptr; + +#ifdef DEBUG + // Zero out data before and after the desired mapping to catch errors early. + if (offset != alignedOffset) + memset(map, 0, offset - alignedOffset); + if (alignedLength % pageSize) + memset(map + alignedLength, 0, pageSize - (alignedLength % pageSize)); +#endif + + return map + (offset - alignedOffset); +} + +void +DeallocateMappedContent(void* p, size_t /*length*/) +{ + if (!p) + return; + + // Calculate the address originally returned by MapViewOfFile. + // This is needed because AllocateMappedContent returns a pointer + // that might be offset from the view, as the beginning of a + // view must be aligned with the allocation granularity. + uintptr_t map = uintptr_t(p) - (uintptr_t(p) % allocGranularity); + MOZ_ALWAYS_TRUE(UnmapViewOfFile(reinterpret_cast<void*>(map))); +} + +# else // Various APIs are unavailable. + +void* +MapAlignedPages(size_t size, size_t alignment) +{ + MOZ_ASSERT(size >= alignment); + MOZ_ASSERT(size >= allocGranularity); + MOZ_ASSERT(size % alignment == 0); + MOZ_ASSERT(size % pageSize == 0); + MOZ_ASSERT_IF(alignment < allocGranularity, allocGranularity % alignment == 0); + MOZ_ASSERT_IF(alignment > allocGranularity, alignment % allocGranularity == 0); + + void* p = _aligned_malloc(size, alignment); + + MOZ_ASSERT(OffsetFromAligned(p, alignment) == 0); + return p; +} + +static void* +MapAlignedPagesLastDitch(size_t size, size_t alignment) +{ + return nullptr; +} + +void +UnmapPages(void* p, size_t size) +{ + _aligned_free(p); +} + +bool +MarkPagesUnused(void* p, size_t size) +{ + MOZ_ASSERT(OffsetFromAligned(p, pageSize) == 0); + return true; +} + +bool +MarkPagesInUse(void* p, size_t size) +{ + MOZ_ASSERT(OffsetFromAligned(p, pageSize) == 0); +} + +size_t +GetPageFaultCount() +{ + // GetProcessMemoryInfo is unavailable. + return 0; +} + +void* +AllocateMappedContent(int fd, size_t offset, size_t length, size_t alignment) +{ + // Not implemented. + return nullptr; +} + +// Deallocate mapped memory for object. +void +DeallocateMappedContent(void* p, size_t length) +{ + // Not implemented. +} + +# endif + +#elif defined(SOLARIS) + +#ifndef MAP_NOSYNC +# define MAP_NOSYNC 0 +#endif + +void +InitMemorySubsystem() +{ + if (pageSize == 0) + pageSize = allocGranularity = size_t(sysconf(_SC_PAGESIZE)); +} + +void* +MapAlignedPages(size_t size, size_t alignment) +{ + MOZ_ASSERT(size >= alignment); + MOZ_ASSERT(size >= allocGranularity); + MOZ_ASSERT(size % alignment == 0); + MOZ_ASSERT(size % pageSize == 0); + MOZ_ASSERT_IF(alignment < allocGranularity, allocGranularity % alignment == 0); + MOZ_ASSERT_IF(alignment > allocGranularity, alignment % allocGranularity == 0); + + int prot = PROT_READ | PROT_WRITE; + int flags = MAP_PRIVATE | MAP_ANON | MAP_ALIGN | MAP_NOSYNC; + + void* p = mmap((caddr_t)alignment, size, prot, flags, -1, 0); + if (p == MAP_FAILED) + return nullptr; + return p; +} + +static void* +MapAlignedPagesLastDitch(size_t size, size_t alignment) +{ + return nullptr; +} + +void +UnmapPages(void* p, size_t size) +{ + MOZ_ALWAYS_TRUE(0 == munmap((caddr_t)p, size)); +} + +bool +MarkPagesUnused(void* p, size_t size) +{ + MOZ_ASSERT(OffsetFromAligned(p, pageSize) == 0); + return true; +} + +bool +MarkPagesInUse(void* p, size_t size) +{ + if (!DecommitEnabled()) + return; + + MOZ_ASSERT(OffsetFromAligned(p, pageSize) == 0); +} + +size_t +GetPageFaultCount() +{ + return 0; +} + +void* +AllocateMappedContent(int fd, size_t offset, size_t length, size_t alignment) +{ + // Not implemented. + return nullptr; +} + +// Deallocate mapped memory for object. +void +DeallocateMappedContent(void* p, size_t length) +{ + // Not implemented. +} + +#elif defined(XP_UNIX) + +void +InitMemorySubsystem() +{ + if (pageSize == 0) + pageSize = allocGranularity = size_t(sysconf(_SC_PAGESIZE)); +} + +static inline void* +MapMemoryAt(void* desired, size_t length, int prot = PROT_READ | PROT_WRITE, + int flags = MAP_PRIVATE | MAP_ANON, int fd = -1, off_t offset = 0) +{ +#if defined(__ia64__) || (defined(__sparc64__) && defined(__NetBSD__)) || defined(__aarch64__) + MOZ_ASSERT((0xffff800000000000ULL & (uintptr_t(desired) + length - 1)) == 0); +#endif + void* region = mmap(desired, length, prot, flags, fd, offset); + if (region == MAP_FAILED) + return nullptr; + /* + * mmap treats the given address as a hint unless the MAP_FIXED flag is + * used (which isn't usually what you want, as this overrides existing + * mappings), so check that the address we got is the address we wanted. + */ + if (region != desired) { + if (munmap(region, length)) + MOZ_ASSERT(errno == ENOMEM); + return nullptr; + } + return region; +} + +static inline void* +MapMemory(size_t length, int prot = PROT_READ | PROT_WRITE, + int flags = MAP_PRIVATE | MAP_ANON, int fd = -1, off_t offset = 0) +{ +#if defined(__ia64__) || (defined(__sparc64__) && defined(__NetBSD__)) + /* + * The JS engine assumes that all allocated pointers have their high 17 bits clear, + * which ia64's mmap doesn't support directly. However, we can emulate it by passing + * mmap an "addr" parameter with those bits clear. The mmap will return that address, + * or the nearest available memory above that address, providing a near-guarantee + * that those bits are clear. If they are not, we return nullptr below to indicate + * out-of-memory. + * + * The addr is chosen as 0x0000070000000000, which still allows about 120TB of virtual + * address space. + * + * See Bug 589735 for more information. + */ + void* region = mmap((void*)0x0000070000000000, length, prot, flags, fd, offset); + if (region == MAP_FAILED) + return nullptr; + /* + * If the allocated memory doesn't have its upper 17 bits clear, consider it + * as out of memory. + */ + if ((uintptr_t(region) + (length - 1)) & 0xffff800000000000) { + if (munmap(region, length)) + MOZ_ASSERT(errno == ENOMEM); + return nullptr; + } + return region; +#elif defined(__aarch64__) + /* + * There might be similar virtual address issue on arm64 which depends on + * hardware and kernel configurations. But the work around is slightly + * different due to the different mmap behavior. + * + * TODO: Merge with the above code block if this implementation works for + * ia64 and sparc64. + */ + const uintptr_t start = UINT64_C(0x0000070000000000); + const uintptr_t end = UINT64_C(0x0000800000000000); + const uintptr_t step = ChunkSize; + /* + * Optimization options if there are too many retries in practice: + * 1. Examine /proc/self/maps to find an available address. This file is + * not always available, however. In addition, even if we examine + * /proc/self/maps, we may still need to retry several times due to + * racing with other threads. + * 2. Use a global/static variable with lock to track the addresses we have + * allocated or tried. + */ + uintptr_t hint; + void* region = MAP_FAILED; + for (hint = start; region == MAP_FAILED && hint + length <= end; hint += step) { + region = mmap((void*)hint, length, prot, flags, fd, offset); + if (region != MAP_FAILED) { + if ((uintptr_t(region) + (length - 1)) & 0xffff800000000000) { + if (munmap(region, length)) { + MOZ_ASSERT(errno == ENOMEM); + } + region = MAP_FAILED; + } + } + } + return region == MAP_FAILED ? nullptr : region; +#else + void* region = MozTaggedAnonymousMmap(nullptr, length, prot, flags, fd, offset, "js-gc-heap"); + if (region == MAP_FAILED) + return nullptr; + return region; +#endif +} + +void* +MapAlignedPages(size_t size, size_t alignment) +{ + MOZ_ASSERT(size >= alignment); + MOZ_ASSERT(size >= allocGranularity); + MOZ_ASSERT(size % alignment == 0); + MOZ_ASSERT(size % pageSize == 0); + MOZ_ASSERT_IF(alignment < allocGranularity, allocGranularity % alignment == 0); + MOZ_ASSERT_IF(alignment > allocGranularity, alignment % allocGranularity == 0); + + void* p = MapMemory(size); + + /* Special case: If we want page alignment, no further work is needed. */ + if (alignment == allocGranularity) + return p; + + if (OffsetFromAligned(p, alignment) == 0) + return p; + + void* retainedAddr; + GetNewChunk(&p, &retainedAddr, size, alignment); + if (retainedAddr) + UnmapPages(retainedAddr, size); + if (p) { + if (OffsetFromAligned(p, alignment) == 0) + return p; + UnmapPages(p, size); + } + + p = MapAlignedPagesSlow(size, alignment); + if (!p) + return MapAlignedPagesLastDitch(size, alignment); + + MOZ_ASSERT(OffsetFromAligned(p, alignment) == 0); + return p; +} + +static void* +MapAlignedPagesSlow(size_t size, size_t alignment) +{ + /* Overallocate and unmap the region's edges. */ + size_t reqSize = size + alignment - pageSize; + void* region = MapMemory(reqSize); + if (!region) + return nullptr; + + void* regionEnd = (void*)(uintptr_t(region) + reqSize); + void* front; + void* end; + if (growthDirection <= 0) { + size_t offset = OffsetFromAligned(regionEnd, alignment); + end = (void*)(uintptr_t(regionEnd) - offset); + front = (void*)(uintptr_t(end) - size); + } else { + size_t offset = OffsetFromAligned(region, alignment); + front = (void*)(uintptr_t(region) + (offset ? alignment - offset : 0)); + end = (void*)(uintptr_t(front) + size); + } + + if (front != region) + UnmapPages(region, uintptr_t(front) - uintptr_t(region)); + if (end != regionEnd) + UnmapPages(end, uintptr_t(regionEnd) - uintptr_t(end)); + + return front; +} + +/* + * In a low memory or high fragmentation situation, alignable chunks of the + * desired size may still be available, even if there are no more contiguous + * free chunks that meet the |size + alignment - pageSize| requirement of + * MapAlignedPagesSlow. In this case, try harder to find an alignable chunk + * by temporarily holding onto the unaligned parts of each chunk until the + * allocator gives us a chunk that either is, or can be aligned. + */ +static void* +MapAlignedPagesLastDitch(size_t size, size_t alignment) +{ + void* tempMaps[MaxLastDitchAttempts]; + int attempt = 0; + void* p = MapMemory(size); + if (OffsetFromAligned(p, alignment) == 0) + return p; + for (; attempt < MaxLastDitchAttempts; ++attempt) { + GetNewChunk(&p, tempMaps + attempt, size, alignment); + if (OffsetFromAligned(p, alignment) == 0) { + if (tempMaps[attempt]) + UnmapPages(tempMaps[attempt], size); + break; + } + if (!tempMaps[attempt]) + break; /* Bail if GetNewChunk failed. */ + } + if (OffsetFromAligned(p, alignment)) { + UnmapPages(p, size); + p = nullptr; + } + while (--attempt >= 0) + UnmapPages(tempMaps[attempt], size); + return p; +} + +/* + * mmap calls don't have to be matched with calls to munmap, so we can unmap + * just the pages we don't need. However, as we don't know a priori if addresses + * are handed out in increasing or decreasing order, we have to try both + * directions (depending on the environment, one will always fail). + */ +static void +GetNewChunk(void** aAddress, void** aRetainedAddr, size_t size, size_t alignment) +{ + void* address = *aAddress; + void* retainedAddr = nullptr; + bool addrsGrowDown = growthDirection <= 0; + int i = 0; + for (; i < 2; ++i) { + /* Try the direction indicated by growthDirection. */ + if (addrsGrowDown) { + size_t offset = OffsetFromAligned(address, alignment); + void* head = (void*)((uintptr_t)address - offset); + void* tail = (void*)((uintptr_t)head + size); + if (MapMemoryAt(head, offset)) { + UnmapPages(tail, offset); + if (growthDirection >= -8) + --growthDirection; + address = head; + break; + } + } else { + size_t offset = alignment - OffsetFromAligned(address, alignment); + void* head = (void*)((uintptr_t)address + offset); + void* tail = (void*)((uintptr_t)address + size); + if (MapMemoryAt(tail, offset)) { + UnmapPages(address, offset); + if (growthDirection <= 8) + ++growthDirection; + address = head; + break; + } + } + /* If we're confident in the growth direction, don't try the other. */ + if (growthDirection < -8 || growthDirection > 8) + break; + /* If that failed, try the opposite direction. */ + addrsGrowDown = !addrsGrowDown; + } + /* If our current chunk cannot be aligned, see if the next one is aligned. */ + if (OffsetFromAligned(address, alignment)) { + retainedAddr = address; + address = MapMemory(size); + } + *aAddress = address; + *aRetainedAddr = retainedAddr; +} + +void +UnmapPages(void* p, size_t size) +{ + if (munmap(p, size)) + MOZ_ASSERT(errno == ENOMEM); +} + +bool +MarkPagesUnused(void* p, size_t size) +{ + if (!DecommitEnabled()) + return false; + + MOZ_ASSERT(OffsetFromAligned(p, pageSize) == 0); + int result = madvise(p, size, MADV_DONTNEED); + return result != -1; +} + +void +MarkPagesInUse(void* p, size_t size) +{ + if (!DecommitEnabled()) + return; + + MOZ_ASSERT(OffsetFromAligned(p, pageSize) == 0); +} + +size_t +GetPageFaultCount() +{ + struct rusage usage; + int err = getrusage(RUSAGE_SELF, &usage); + if (err) + return 0; + return usage.ru_majflt; +} + +void* +AllocateMappedContent(int fd, size_t offset, size_t length, size_t alignment) +{ + MOZ_ASSERT(length && alignment); + + // The allocation granularity and the requested offset + // must both be divisible by the requested alignment. + // Alignments larger than the allocation granularity are not supported. + if (allocGranularity % alignment != 0 || offset % alignment != 0) + return nullptr; + + // Sanity check the offset and size, as mmap does not do this for us. + struct stat st; + if (fstat(fd, &st) || offset >= uint64_t(st.st_size) || length > uint64_t(st.st_size) - offset) + return nullptr; + + size_t alignedOffset = offset - (offset % allocGranularity); + size_t alignedLength = length + (offset % allocGranularity); + + uint8_t* map = static_cast<uint8_t*>(MapMemory(alignedLength, PROT_READ | PROT_WRITE, + MAP_PRIVATE, fd, alignedOffset)); + if (!map) + return nullptr; + +#ifdef DEBUG + // Zero out data before and after the desired mapping to catch errors early. + if (offset != alignedOffset) + memset(map, 0, offset - alignedOffset); + if (alignedLength % pageSize) + memset(map + alignedLength, 0, pageSize - (alignedLength % pageSize)); +#endif + + return map + (offset - alignedOffset); +} + +void +DeallocateMappedContent(void* p, size_t length) +{ + if (!p) + return; + + // Calculate the address originally returned by mmap. + // This is needed because AllocateMappedContent returns a pointer + // that might be offset from the mapping, as the beginning of a + // mapping must be aligned with the allocation granularity. + uintptr_t map = uintptr_t(p) - (uintptr_t(p) % allocGranularity); + size_t alignedLength = length + (uintptr_t(p) % allocGranularity); + UnmapPages(reinterpret_cast<void*>(map), alignedLength); +} + +#else +#error "Memory mapping functions are not defined for your OS." +#endif + +void +ProtectPages(void* p, size_t size) +{ + MOZ_ASSERT(size % pageSize == 0); + MOZ_RELEASE_ASSERT(size > 0); + MOZ_RELEASE_ASSERT(p); +#if defined(XP_WIN) + DWORD oldProtect; + if (!VirtualProtect(p, size, PAGE_NOACCESS, &oldProtect)) { + MOZ_CRASH_UNSAFE_PRINTF("VirtualProtect(PAGE_NOACCESS) failed! Error code: %u", + GetLastError()); + } + MOZ_ASSERT(oldProtect == PAGE_READWRITE); +#else // assume Unix + if (mprotect(p, size, PROT_NONE)) + MOZ_CRASH("mprotect(PROT_NONE) failed"); +#endif +} + +void +MakePagesReadOnly(void* p, size_t size) +{ + MOZ_ASSERT(size % pageSize == 0); + MOZ_RELEASE_ASSERT(size > 0); + MOZ_RELEASE_ASSERT(p); +#if defined(XP_WIN) + DWORD oldProtect; + if (!VirtualProtect(p, size, PAGE_READONLY, &oldProtect)) { + MOZ_CRASH_UNSAFE_PRINTF("VirtualProtect(PAGE_READONLY) failed! Error code: %u", + GetLastError()); + } + MOZ_ASSERT(oldProtect == PAGE_READWRITE); +#else // assume Unix + if (mprotect(p, size, PROT_READ)) + MOZ_CRASH("mprotect(PROT_READ) failed"); +#endif +} + +void +UnprotectPages(void* p, size_t size) +{ + MOZ_ASSERT(size % pageSize == 0); + MOZ_RELEASE_ASSERT(size > 0); + MOZ_RELEASE_ASSERT(p); +#if defined(XP_WIN) + DWORD oldProtect; + if (!VirtualProtect(p, size, PAGE_READWRITE, &oldProtect)) { + MOZ_CRASH_UNSAFE_PRINTF("VirtualProtect(PAGE_READWRITE) failed! Error code: %u", + GetLastError()); + } + MOZ_ASSERT(oldProtect == PAGE_NOACCESS || oldProtect == PAGE_READONLY); +#else // assume Unix + if (mprotect(p, size, PROT_READ | PROT_WRITE)) + MOZ_CRASH("mprotect(PROT_READ | PROT_WRITE) failed"); +#endif +} + +} // namespace gc +} // namespace js diff --git a/js/src/gc/Memory.h b/js/src/gc/Memory.h new file mode 100644 index 000000000..4ff7bff4c --- /dev/null +++ b/js/src/gc/Memory.h @@ -0,0 +1,53 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_Memory_h +#define gc_Memory_h + +#include <stddef.h> + +namespace js { +namespace gc { + +// Sanity check that our compiled configuration matches the currently +// running instance and initialize any runtime data needed for allocation. +void InitMemorySubsystem(); + +size_t SystemPageSize(); + +// Allocate or deallocate pages from the system with the given alignment. +void* MapAlignedPages(size_t size, size_t alignment); +void UnmapPages(void* p, size_t size); + +// Tell the OS that the given pages are not in use, so they should not be +// written to a paging file. This may be a no-op on some platforms. +bool MarkPagesUnused(void* p, size_t size); + +// Undo |MarkPagesUnused|: tell the OS that the given pages are of interest +// and should be paged in and out normally. This may be a no-op on some +// platforms. +void MarkPagesInUse(void* p, size_t size); + +// Returns #(hard faults) + #(soft faults) +size_t GetPageFaultCount(); + +// Allocate memory mapped content. +// The offset must be aligned according to alignment requirement. +void* AllocateMappedContent(int fd, size_t offset, size_t length, size_t alignment); + +// Deallocate memory mapped content. +void DeallocateMappedContent(void* p, size_t length); + +void* TestMapAlignedPagesLastDitch(size_t size, size_t alignment); + +void ProtectPages(void* p, size_t size); +void MakePagesReadOnly(void* p, size_t size); +void UnprotectPages(void* p, size_t size); + +} // namespace gc +} // namespace js + +#endif /* gc_Memory_h */ diff --git a/js/src/gc/MemoryProfiler.cpp b/js/src/gc/MemoryProfiler.cpp new file mode 100644 index 000000000..a061f9303 --- /dev/null +++ b/js/src/gc/MemoryProfiler.cpp @@ -0,0 +1,49 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "jscntxt.h" +#include "jsfriendapi.h" + +#include "vm/Runtime.h" + +using js::gc::Cell; + +mozilla::Atomic<uint32_t, mozilla::Relaxed> MemProfiler::sActiveProfilerCount; +NativeProfiler* MemProfiler::sNativeProfiler; + +GCHeapProfiler* +MemProfiler::GetGCHeapProfiler(void* addr) +{ + JSRuntime* runtime = reinterpret_cast<Cell*>(addr)->runtimeFromAnyThread(); + return runtime->gc.mMemProfiler.mGCHeapProfiler; +} + +GCHeapProfiler* +MemProfiler::GetGCHeapProfiler(JSRuntime* runtime) +{ + return runtime->gc.mMemProfiler.mGCHeapProfiler; +} + +JS_FRIEND_API(MemProfiler*) +MemProfiler::GetMemProfiler(JSContext* context) +{ + return &context->gc.mMemProfiler; +} + +JS_FRIEND_API(void) +MemProfiler::start(GCHeapProfiler* aGCHeapProfiler) +{ + ReleaseAllJITCode(mRuntime->defaultFreeOp()); + mGCHeapProfiler = aGCHeapProfiler; + sActiveProfilerCount++; +} + +JS_FRIEND_API(void) +MemProfiler::stop() +{ + sActiveProfilerCount--; + mGCHeapProfiler = nullptr; +} diff --git a/js/src/gc/Nursery-inl.h b/js/src/gc/Nursery-inl.h new file mode 100644 index 000000000..8ebfd8232 --- /dev/null +++ b/js/src/gc/Nursery-inl.h @@ -0,0 +1,88 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=4 sw=4 et tw=79 ft=cpp: + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_Nursery_inl_h +#define gc_Nursery_inl_h + +#include "gc/Nursery.h" + +#include "jscntxt.h" + +#include "gc/Heap.h" +#include "gc/Zone.h" +#include "js/TracingAPI.h" +#include "vm/Runtime.h" + +MOZ_ALWAYS_INLINE bool +js::Nursery::getForwardedPointer(JSObject** ref) const +{ + MOZ_ASSERT(ref); + MOZ_ASSERT(isInside((void*)*ref)); + const gc::RelocationOverlay* overlay = reinterpret_cast<const gc::RelocationOverlay*>(*ref); + if (!overlay->isForwarded()) + return false; + *ref = static_cast<JSObject*>(overlay->forwardingAddress()); + return true; +} + +namespace js { + +// The allocation methods below will not run the garbage collector. If the +// nursery cannot accomodate the allocation, the malloc heap will be used +// instead. + +template <typename T> +static inline T* +AllocateObjectBuffer(ExclusiveContext* cx, uint32_t count) +{ + if (cx->isJSContext()) { + Nursery& nursery = cx->asJSContext()->runtime()->gc.nursery; + size_t nbytes = JS_ROUNDUP(count * sizeof(T), sizeof(Value)); + T* buffer = static_cast<T*>(nursery.allocateBuffer(cx->zone(), nbytes)); + if (!buffer) + ReportOutOfMemory(cx); + return buffer; + } + return cx->zone()->pod_malloc<T>(count); +} + +template <typename T> +static inline T* +AllocateObjectBuffer(ExclusiveContext* cx, JSObject* obj, uint32_t count) +{ + if (cx->isJSContext()) { + Nursery& nursery = cx->asJSContext()->runtime()->gc.nursery; + size_t nbytes = JS_ROUNDUP(count * sizeof(T), sizeof(Value)); + T* buffer = static_cast<T*>(nursery.allocateBuffer(obj, nbytes)); + if (!buffer) + ReportOutOfMemory(cx); + return buffer; + } + return obj->zone()->pod_malloc<T>(count); +} + +// If this returns null then the old buffer will be left alone. +template <typename T> +static inline T* +ReallocateObjectBuffer(ExclusiveContext* cx, JSObject* obj, T* oldBuffer, + uint32_t oldCount, uint32_t newCount) +{ + if (cx->isJSContext()) { + Nursery& nursery = cx->asJSContext()->runtime()->gc.nursery; + T* buffer = static_cast<T*>(nursery.reallocateBuffer(obj, oldBuffer, + oldCount * sizeof(T), + newCount * sizeof(T))); + if (!buffer) + ReportOutOfMemory(cx); + return buffer; + } + return obj->zone()->pod_realloc<T>(oldBuffer, oldCount, newCount); +} + +} // namespace js + +#endif /* gc_Nursery_inl_h */ diff --git a/js/src/gc/Nursery.cpp b/js/src/gc/Nursery.cpp new file mode 100644 index 000000000..2c402fe0b --- /dev/null +++ b/js/src/gc/Nursery.cpp @@ -0,0 +1,1025 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sw=4 et tw=78: + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/Nursery-inl.h" + +#include "mozilla/DebugOnly.h" +#include "mozilla/IntegerPrintfMacros.h" +#include "mozilla/Move.h" +#include "mozilla/Unused.h" + +#include "jscompartment.h" +#include "jsfriendapi.h" +#include "jsgc.h" +#include "jsutil.h" + +#include "gc/GCInternals.h" +#include "gc/Memory.h" +#include "jit/JitFrames.h" +#include "vm/ArrayObject.h" +#include "vm/Debugger.h" +#if defined(DEBUG) +#include "vm/EnvironmentObject.h" +#endif +#include "vm/Time.h" +#include "vm/TypedArrayObject.h" +#include "vm/TypeInference.h" + +#include "jsobjinlines.h" + +#include "vm/NativeObject-inl.h" + +using namespace js; +using namespace gc; + +using mozilla::ArrayLength; +using mozilla::DebugOnly; +using mozilla::PodCopy; +using mozilla::PodZero; + +static const uintptr_t CanaryMagicValue = 0xDEADB15D; + +struct js::Nursery::FreeMallocedBuffersTask : public GCParallelTask +{ + explicit FreeMallocedBuffersTask(FreeOp* fop) : fop_(fop) {} + bool init() { return buffers_.init(); } + void transferBuffersToFree(MallocedBuffersSet& buffersToFree, + const AutoLockHelperThreadState& lock); + ~FreeMallocedBuffersTask() override { join(); } + + private: + FreeOp* fop_; + MallocedBuffersSet buffers_; + + virtual void run() override; +}; + +struct js::Nursery::SweepAction +{ + SweepAction(SweepThunk thunk, void* data, SweepAction* next) + : thunk(thunk), data(data), next(next) + {} + + SweepThunk thunk; + void* data; + SweepAction* next; + +#if JS_BITS_PER_WORD == 32 + protected: + uint32_t padding; +#endif +}; + +#ifdef JS_GC_ZEAL +struct js::Nursery::Canary +{ + uintptr_t magicValue; + Canary* next; +}; +#endif + +inline void +js::Nursery::NurseryChunk::poisonAndInit(JSRuntime* rt, uint8_t poison) +{ + JS_POISON(this, poison, ChunkSize); + init(rt); +} + +inline void +js::Nursery::NurseryChunk::init(JSRuntime* rt) +{ + new (&trailer) gc::ChunkTrailer(rt, &rt->gc.storeBuffer); +} + +/* static */ inline js::Nursery::NurseryChunk* +js::Nursery::NurseryChunk::fromChunk(Chunk* chunk) +{ + return reinterpret_cast<NurseryChunk*>(chunk); +} + +inline Chunk* +js::Nursery::NurseryChunk::toChunk(JSRuntime* rt) +{ + auto chunk = reinterpret_cast<Chunk*>(this); + chunk->init(rt); + return chunk; +} + +js::Nursery::Nursery(JSRuntime* rt) + : runtime_(rt) + , position_(0) + , currentStartChunk_(0) + , currentStartPosition_(0) + , currentEnd_(0) + , currentChunk_(0) + , maxNurseryChunks_(0) + , previousPromotionRate_(0) + , profileThreshold_(0) + , enableProfiling_(false) + , reportTenurings_(0) + , minorGcCount_(0) + , freeMallocedBuffersTask(nullptr) + , sweepActions_(nullptr) +#ifdef JS_GC_ZEAL + , lastCanary_(nullptr) +#endif +{} + +bool +js::Nursery::init(uint32_t maxNurseryBytes, AutoLockGC& lock) +{ + /* maxNurseryBytes parameter is rounded down to a multiple of chunk size. */ + maxNurseryChunks_ = maxNurseryBytes >> ChunkShift; + + /* If no chunks are specified then the nursery is permenantly disabled. */ + if (maxNurseryChunks_ == 0) + return true; + + if (!mallocedBuffers.init()) + return false; + + if (!cellsWithUid_.init()) + return false; + + freeMallocedBuffersTask = js_new<FreeMallocedBuffersTask>(runtime()->defaultFreeOp()); + if (!freeMallocedBuffersTask || !freeMallocedBuffersTask->init()) + return false; + + AutoMaybeStartBackgroundAllocation maybeBgAlloc; + updateNumChunksLocked(1, maybeBgAlloc, lock); + if (numChunks() == 0) + return false; + + setCurrentChunk(0); + setStartPosition(); + + char* env = getenv("JS_GC_PROFILE_NURSERY"); + if (env) { + if (0 == strcmp(env, "help")) { + fprintf(stderr, "JS_GC_PROFILE_NURSERY=N\n" + "\tReport minor GC's taking at least N microseconds.\n"); + exit(0); + } + enableProfiling_ = true; + profileThreshold_ = atoi(env); + } + + env = getenv("JS_GC_REPORT_TENURING"); + if (env) { + if (0 == strcmp(env, "help")) { + fprintf(stderr, "JS_GC_REPORT_TENURING=N\n" + "\tAfter a minor GC, report any ObjectGroups with at least N instances tenured.\n"); + exit(0); + } + reportTenurings_ = atoi(env); + } + + PodZero(&startTimes_); + PodZero(&profileTimes_); + PodZero(&totalTimes_); + + if (!runtime()->gc.storeBuffer.enable()) + return false; + + MOZ_ASSERT(isEnabled()); + return true; +} + +js::Nursery::~Nursery() +{ + disable(); + js_delete(freeMallocedBuffersTask); +} + +void +js::Nursery::enable() +{ + MOZ_ASSERT(isEmpty()); + MOZ_ASSERT(!runtime()->gc.isVerifyPreBarriersEnabled()); + if (isEnabled()) + return; + + updateNumChunks(1); + if (numChunks() == 0) + return; + + setCurrentChunk(0); + setStartPosition(); +#ifdef JS_GC_ZEAL + if (runtime()->hasZealMode(ZealMode::GenerationalGC)) + enterZealMode(); +#endif + + MOZ_ALWAYS_TRUE(runtime()->gc.storeBuffer.enable()); + return; +} + +void +js::Nursery::disable() +{ + MOZ_ASSERT(isEmpty()); + if (!isEnabled()) + return; + updateNumChunks(0); + currentEnd_ = 0; + runtime()->gc.storeBuffer.disable(); +} + +bool +js::Nursery::isEmpty() const +{ + MOZ_ASSERT(runtime_); + if (!isEnabled()) + return true; + + if (!runtime_->hasZealMode(ZealMode::GenerationalGC)) { + MOZ_ASSERT(currentStartChunk_ == 0); + MOZ_ASSERT(currentStartPosition_ == chunk(0).start()); + } + return position() == currentStartPosition_; +} + +#ifdef JS_GC_ZEAL +void +js::Nursery::enterZealMode() { + if (isEnabled()) + updateNumChunks(maxNurseryChunks_); +} + +void +js::Nursery::leaveZealMode() { + if (isEnabled()) { + MOZ_ASSERT(isEmpty()); + setCurrentChunk(0); + setStartPosition(); + } +} +#endif // JS_GC_ZEAL + +JSObject* +js::Nursery::allocateObject(JSContext* cx, size_t size, size_t numDynamic, const js::Class* clasp) +{ + /* Ensure there's enough space to replace the contents with a RelocationOverlay. */ + MOZ_ASSERT(size >= sizeof(RelocationOverlay)); + + /* Sanity check the finalizer. */ + MOZ_ASSERT_IF(clasp->hasFinalize(), CanNurseryAllocateFinalizedClass(clasp) || + clasp->isProxy()); + + /* Make the object allocation. */ + JSObject* obj = static_cast<JSObject*>(allocate(size)); + if (!obj) + return nullptr; + + /* If we want external slots, add them. */ + HeapSlot* slots = nullptr; + if (numDynamic) { + MOZ_ASSERT(clasp->isNative() || clasp->isProxy()); + slots = static_cast<HeapSlot*>(allocateBuffer(cx->zone(), numDynamic * sizeof(HeapSlot))); + if (!slots) { + /* + * It is safe to leave the allocated object uninitialized, since we + * do not visit unallocated things in the nursery. + */ + return nullptr; + } + } + + /* Always initialize the slots field to match the JIT behavior. */ + obj->setInitialSlotsMaybeNonNative(slots); + + TraceNurseryAlloc(obj, size); + return obj; +} + +void* +js::Nursery::allocate(size_t size) +{ + MOZ_ASSERT(isEnabled()); + MOZ_ASSERT(!runtime()->isHeapBusy()); + MOZ_ASSERT_IF(currentChunk_ == currentStartChunk_, position() >= currentStartPosition_); + MOZ_ASSERT(position() % gc::CellSize == 0); + MOZ_ASSERT(size % gc::CellSize == 0); + +#ifdef JS_GC_ZEAL + static const size_t CanarySize = (sizeof(Nursery::Canary) + CellSize - 1) & ~CellMask; + if (runtime()->gc.hasZealMode(ZealMode::CheckNursery)) + size += CanarySize; +#endif + + if (currentEnd() < position() + size) { + if (currentChunk_ + 1 == numChunks()) + return nullptr; + setCurrentChunk(currentChunk_ + 1); + } + + void* thing = (void*)position(); + position_ = position() + size; + + JS_EXTRA_POISON(thing, JS_ALLOCATED_NURSERY_PATTERN, size); + +#ifdef JS_GC_ZEAL + if (runtime()->gc.hasZealMode(ZealMode::CheckNursery)) { + auto canary = reinterpret_cast<Canary*>(position() - CanarySize); + canary->magicValue = CanaryMagicValue; + canary->next = nullptr; + if (lastCanary_) { + MOZ_ASSERT(!lastCanary_->next); + lastCanary_->next = canary; + } + lastCanary_ = canary; + } +#endif + + MemProfiler::SampleNursery(reinterpret_cast<void*>(thing), size); + return thing; +} + +void* +js::Nursery::allocateBuffer(Zone* zone, size_t nbytes) +{ + MOZ_ASSERT(nbytes > 0); + + if (nbytes <= MaxNurseryBufferSize) { + void* buffer = allocate(nbytes); + if (buffer) + return buffer; + } + + void* buffer = zone->pod_malloc<uint8_t>(nbytes); + if (buffer && !mallocedBuffers.putNew(buffer)) { + js_free(buffer); + return nullptr; + } + return buffer; +} + +void* +js::Nursery::allocateBuffer(JSObject* obj, size_t nbytes) +{ + MOZ_ASSERT(obj); + MOZ_ASSERT(nbytes > 0); + + if (!IsInsideNursery(obj)) + return obj->zone()->pod_malloc<uint8_t>(nbytes); + return allocateBuffer(obj->zone(), nbytes); +} + +void* +js::Nursery::reallocateBuffer(JSObject* obj, void* oldBuffer, + size_t oldBytes, size_t newBytes) +{ + if (!IsInsideNursery(obj)) + return obj->zone()->pod_realloc<uint8_t>((uint8_t*)oldBuffer, oldBytes, newBytes); + + if (!isInside(oldBuffer)) { + void* newBuffer = obj->zone()->pod_realloc<uint8_t>((uint8_t*)oldBuffer, oldBytes, newBytes); + if (newBuffer && oldBuffer != newBuffer) + MOZ_ALWAYS_TRUE(mallocedBuffers.rekeyAs(oldBuffer, newBuffer, newBuffer)); + return newBuffer; + } + + /* The nursery cannot make use of the returned slots data. */ + if (newBytes < oldBytes) + return oldBuffer; + + void* newBuffer = allocateBuffer(obj->zone(), newBytes); + if (newBuffer) + PodCopy((uint8_t*)newBuffer, (uint8_t*)oldBuffer, oldBytes); + return newBuffer; +} + +void +js::Nursery::freeBuffer(void* buffer) +{ + if (!isInside(buffer)) { + removeMallocedBuffer(buffer); + js_free(buffer); + } +} + +void +Nursery::setForwardingPointer(void* oldData, void* newData, bool direct) +{ + MOZ_ASSERT(isInside(oldData)); + + // Bug 1196210: If a zero-capacity header lands in the last 2 words of a + // jemalloc chunk abutting the start of a nursery chunk, the (invalid) + // newData pointer will appear to be "inside" the nursery. + MOZ_ASSERT(!isInside(newData) || (uintptr_t(newData) & ChunkMask) == 0); + + if (direct) { + *reinterpret_cast<void**>(oldData) = newData; + } else { + AutoEnterOOMUnsafeRegion oomUnsafe; + if (!forwardedBuffers.initialized() && !forwardedBuffers.init()) + oomUnsafe.crash("Nursery::setForwardingPointer"); +#ifdef DEBUG + if (ForwardedBufferMap::Ptr p = forwardedBuffers.lookup(oldData)) + MOZ_ASSERT(p->value() == newData); +#endif + if (!forwardedBuffers.put(oldData, newData)) + oomUnsafe.crash("Nursery::setForwardingPointer"); + } +} + +void +Nursery::setSlotsForwardingPointer(HeapSlot* oldSlots, HeapSlot* newSlots, uint32_t nslots) +{ + // Slot arrays always have enough space for a forwarding pointer, since the + // number of slots is never zero. + MOZ_ASSERT(nslots > 0); + setForwardingPointer(oldSlots, newSlots, /* direct = */ true); +} + +void +Nursery::setElementsForwardingPointer(ObjectElements* oldHeader, ObjectElements* newHeader, + uint32_t nelems) +{ + // Only use a direct forwarding pointer if there is enough space for one. + setForwardingPointer(oldHeader->elements(), newHeader->elements(), + nelems > ObjectElements::VALUES_PER_HEADER); +} + +#ifdef DEBUG +static bool IsWriteableAddress(void* ptr) +{ + volatile uint64_t* vPtr = reinterpret_cast<volatile uint64_t*>(ptr); + *vPtr = *vPtr; + return true; +} +#endif + +void +js::Nursery::forwardBufferPointer(HeapSlot** pSlotsElems) +{ + HeapSlot* old = *pSlotsElems; + + if (!isInside(old)) + return; + + // The new location for this buffer is either stored inline with it or in + // the forwardedBuffers table. + do { + if (forwardedBuffers.initialized()) { + if (ForwardedBufferMap::Ptr p = forwardedBuffers.lookup(old)) { + *pSlotsElems = reinterpret_cast<HeapSlot*>(p->value()); + break; + } + } + + *pSlotsElems = *reinterpret_cast<HeapSlot**>(old); + } while (false); + + MOZ_ASSERT(!isInside(*pSlotsElems)); + MOZ_ASSERT(IsWriteableAddress(*pSlotsElems)); +} + +js::TenuringTracer::TenuringTracer(JSRuntime* rt, Nursery* nursery) + : JSTracer(rt, JSTracer::TracerKindTag::Tenuring, TraceWeakMapKeysValues) + , nursery_(*nursery) + , tenuredSize(0) + , head(nullptr) + , tail(&head) +{ +} + +/* static */ void +js::Nursery::printProfileHeader() +{ +#define PRINT_HEADER(name, text) \ + fprintf(stderr, " %6s", text); +FOR_EACH_NURSERY_PROFILE_TIME(PRINT_HEADER) +#undef PRINT_HEADER + fprintf(stderr, "\n"); +} + +/* static */ void +js::Nursery::printProfileTimes(const ProfileTimes& times) +{ + for (auto time : times) + fprintf(stderr, " %6" PRIi64, time); + fprintf(stderr, "\n"); +} + +void +js::Nursery::printTotalProfileTimes() +{ + if (enableProfiling_) { + fprintf(stderr, "MinorGC TOTALS: %7" PRIu64 " collections: ", minorGcCount_); + printProfileTimes(totalTimes_); + } +} + +inline void +js::Nursery::startProfile(ProfileKey key) +{ + startTimes_[key] = PRMJ_Now(); +} + +inline void +js::Nursery::endProfile(ProfileKey key) +{ + profileTimes_[key] = PRMJ_Now() - startTimes_[key]; + totalTimes_[key] += profileTimes_[key]; +} + +inline void +js::Nursery::maybeStartProfile(ProfileKey key) +{ + if (enableProfiling_) + startProfile(key); +} + +inline void +js::Nursery::maybeEndProfile(ProfileKey key) +{ + if (enableProfiling_) + endProfile(key); +} + +void +js::Nursery::collect(JSRuntime* rt, JS::gcreason::Reason reason) +{ + MOZ_ASSERT(!rt->mainThread.suppressGC); + MOZ_RELEASE_ASSERT(CurrentThreadCanAccessRuntime(rt)); + + if (!isEnabled() || isEmpty()) { + // Our barriers are not always exact, and there may be entries in the + // storebuffer even when the nursery is disabled or empty. It's not safe + // to keep these entries as they may refer to tenured cells which may be + // freed after this point. + rt->gc.storeBuffer.clear(); + } + + if (!isEnabled()) + return; + + rt->gc.incMinorGcNumber(); + +#ifdef JS_GC_ZEAL + if (rt->gc.hasZealMode(ZealMode::CheckNursery)) { + for (auto canary = lastCanary_; canary; canary = canary->next) + MOZ_ASSERT(canary->magicValue == CanaryMagicValue); + } + lastCanary_ = nullptr; +#endif + + rt->gc.stats.beginNurseryCollection(reason); + TraceMinorGCStart(); + + startProfile(ProfileKey::Total); + + // The hazard analysis thinks doCollection can invalidate pointers in + // tenureCounts below. + JS::AutoSuppressGCAnalysis nogc; + + TenureCountCache tenureCounts; + double promotionRate = 0; + if (!isEmpty()) + promotionRate = doCollection(rt, reason, tenureCounts); + + // Resize the nursery. + maybeStartProfile(ProfileKey::Resize); + maybeResizeNursery(reason, promotionRate); + maybeEndProfile(ProfileKey::Resize); + + // If we are promoting the nursery, or exhausted the store buffer with + // pointers to nursery things, which will force a collection well before + // the nursery is full, look for object groups that are getting promoted + // excessively and try to pretenure them. + maybeStartProfile(ProfileKey::Pretenure); + uint32_t pretenureCount = 0; + if (promotionRate > 0.8 || reason == JS::gcreason::FULL_STORE_BUFFER) { + JSContext* cx = rt->contextFromMainThread(); + for (auto& entry : tenureCounts.entries) { + if (entry.count >= 3000) { + ObjectGroup* group = entry.group; + if (group->canPreTenure()) { + AutoCompartment ac(cx, group->compartment()); + group->setShouldPreTenure(cx); + pretenureCount++; + } + } + } + } + maybeEndProfile(ProfileKey::Pretenure); + + // We ignore gcMaxBytes when allocating for minor collection. However, if we + // overflowed, we disable the nursery. The next time we allocate, we'll fail + // because gcBytes >= gcMaxBytes. + if (rt->gc.usage.gcBytes() >= rt->gc.tunables.gcMaxBytes()) + disable(); + + endProfile(ProfileKey::Total); + minorGcCount_++; + + int64_t totalTime = profileTimes_[ProfileKey::Total]; + rt->addTelemetry(JS_TELEMETRY_GC_MINOR_US, totalTime); + rt->addTelemetry(JS_TELEMETRY_GC_MINOR_REASON, reason); + if (totalTime > 1000) + rt->addTelemetry(JS_TELEMETRY_GC_MINOR_REASON_LONG, reason); + rt->addTelemetry(JS_TELEMETRY_GC_NURSERY_BYTES, sizeOfHeapCommitted()); + rt->addTelemetry(JS_TELEMETRY_GC_PRETENURE_COUNT, pretenureCount); + + rt->gc.stats.endNurseryCollection(reason); + TraceMinorGCEnd(); + + if (enableProfiling_ && totalTime >= profileThreshold_) { + static int printedHeader = 0; + if ((printedHeader++ % 200) == 0) { + fprintf(stderr, "MinorGC: Reason PRate Size "); + printProfileHeader(); + } + + fprintf(stderr, "MinorGC: %20s %5.1f%% %4u ", + JS::gcreason::ExplainReason(reason), + promotionRate * 100, + numChunks()); + printProfileTimes(profileTimes_); + + if (reportTenurings_) { + for (auto& entry : tenureCounts.entries) { + if (entry.count >= reportTenurings_) { + fprintf(stderr, "%d x ", entry.count); + entry.group->print(); + } + } + } + } +} + +double +js::Nursery::doCollection(JSRuntime* rt, JS::gcreason::Reason reason, + TenureCountCache& tenureCounts) +{ + AutoTraceSession session(rt, JS::HeapState::MinorCollecting); + AutoSetThreadIsPerformingGC performingGC; + AutoStopVerifyingBarriers av(rt, false); + AutoDisableProxyCheck disableStrictProxyChecking(rt); + mozilla::DebugOnly<AutoEnterOOMUnsafeRegion> oomUnsafeRegion; + + size_t initialNurserySize = spaceToEnd(); + + // Move objects pointed to by roots from the nursery to the major heap. + TenuringTracer mover(rt, this); + + // Mark the store buffer. This must happen first. + StoreBuffer& sb = rt->gc.storeBuffer; + + // The MIR graph only contains nursery pointers if cancelIonCompilations() + // is set on the store buffer, in which case we cancel all compilations. + maybeStartProfile(ProfileKey::CancelIonCompilations); + if (sb.cancelIonCompilations()) + js::CancelOffThreadIonCompile(rt); + maybeEndProfile(ProfileKey::CancelIonCompilations); + + maybeStartProfile(ProfileKey::TraceValues); + sb.traceValues(mover); + maybeEndProfile(ProfileKey::TraceValues); + + maybeStartProfile(ProfileKey::TraceCells); + sb.traceCells(mover); + maybeEndProfile(ProfileKey::TraceCells); + + maybeStartProfile(ProfileKey::TraceSlots); + sb.traceSlots(mover); + maybeEndProfile(ProfileKey::TraceSlots); + + maybeStartProfile(ProfileKey::TraceWholeCells); + sb.traceWholeCells(mover); + maybeEndProfile(ProfileKey::TraceWholeCells); + + maybeStartProfile(ProfileKey::TraceGenericEntries); + sb.traceGenericEntries(&mover); + maybeEndProfile(ProfileKey::TraceGenericEntries); + + maybeStartProfile(ProfileKey::MarkRuntime); + rt->gc.traceRuntimeForMinorGC(&mover, session.lock); + maybeEndProfile(ProfileKey::MarkRuntime); + + maybeStartProfile(ProfileKey::MarkDebugger); + { + gcstats::AutoPhase ap(rt->gc.stats, gcstats::PHASE_MARK_ROOTS); + Debugger::markAll(&mover); + } + maybeEndProfile(ProfileKey::MarkDebugger); + + maybeStartProfile(ProfileKey::ClearNewObjectCache); + rt->contextFromMainThread()->caches.newObjectCache.clearNurseryObjects(rt); + maybeEndProfile(ProfileKey::ClearNewObjectCache); + + // Most of the work is done here. This loop iterates over objects that have + // been moved to the major heap. If these objects have any outgoing pointers + // to the nursery, then those nursery objects get moved as well, until no + // objects are left to move. That is, we iterate to a fixed point. + maybeStartProfile(ProfileKey::CollectToFP); + collectToFixedPoint(mover, tenureCounts); + maybeEndProfile(ProfileKey::CollectToFP); + + // Sweep compartments to update the array buffer object's view lists. + maybeStartProfile(ProfileKey::SweepArrayBufferViewList); + for (CompartmentsIter c(rt, SkipAtoms); !c.done(); c.next()) + c->sweepAfterMinorGC(&mover); + maybeEndProfile(ProfileKey::SweepArrayBufferViewList); + + // Update any slot or element pointers whose destination has been tenured. + maybeStartProfile(ProfileKey::UpdateJitActivations); + js::jit::UpdateJitActivationsForMinorGC(rt, &mover); + forwardedBuffers.finish(); + maybeEndProfile(ProfileKey::UpdateJitActivations); + + maybeStartProfile(ProfileKey::ObjectsTenuredCallback); + rt->gc.callObjectsTenuredCallback(); + maybeEndProfile(ProfileKey::ObjectsTenuredCallback); + + // Sweep. + maybeStartProfile(ProfileKey::FreeMallocedBuffers); + freeMallocedBuffers(); + maybeEndProfile(ProfileKey::FreeMallocedBuffers); + + maybeStartProfile(ProfileKey::Sweep); + sweep(); + maybeEndProfile(ProfileKey::Sweep); + + maybeStartProfile(ProfileKey::ClearStoreBuffer); + rt->gc.storeBuffer.clear(); + maybeEndProfile(ProfileKey::ClearStoreBuffer); + + // Make sure hashtables have been updated after the collection. + maybeStartProfile(ProfileKey::CheckHashTables); +#ifdef JS_GC_ZEAL + if (rt->hasZealMode(ZealMode::CheckHashTablesOnMinorGC)) + CheckHashTablesAfterMovingGC(rt); +#endif + maybeEndProfile(ProfileKey::CheckHashTables); + + // Calculate and return the promotion rate. + return mover.tenuredSize / double(initialNurserySize); +} + +void +js::Nursery::FreeMallocedBuffersTask::transferBuffersToFree(MallocedBuffersSet& buffersToFree, + const AutoLockHelperThreadState& lock) +{ + // Transfer the contents of the source set to the task's buffers_ member by + // swapping the sets, which also clears the source. + MOZ_ASSERT(!isRunningWithLockHeld(lock)); + MOZ_ASSERT(buffers_.empty()); + mozilla::Swap(buffers_, buffersToFree); +} + +void +js::Nursery::FreeMallocedBuffersTask::run() +{ + for (MallocedBuffersSet::Range r = buffers_.all(); !r.empty(); r.popFront()) + fop_->free_(r.front()); + buffers_.clear(); +} + +void +js::Nursery::freeMallocedBuffers() +{ + if (mallocedBuffers.empty()) + return; + + bool started; + { + AutoLockHelperThreadState lock; + freeMallocedBuffersTask->joinWithLockHeld(lock); + freeMallocedBuffersTask->transferBuffersToFree(mallocedBuffers, lock); + started = freeMallocedBuffersTask->startWithLockHeld(lock); + } + + if (!started) + freeMallocedBuffersTask->runFromMainThread(runtime()); + + MOZ_ASSERT(mallocedBuffers.empty()); +} + +void +js::Nursery::waitBackgroundFreeEnd() +{ + // We may finishRoots before nursery init if runtime init fails. + if (!isEnabled()) + return; + + MOZ_ASSERT(freeMallocedBuffersTask); + freeMallocedBuffersTask->join(); +} + +void +js::Nursery::sweep() +{ + /* Sweep unique id's in all in-use chunks. */ + for (CellsWithUniqueIdSet::Enum e(cellsWithUid_); !e.empty(); e.popFront()) { + JSObject* obj = static_cast<JSObject*>(e.front()); + if (!IsForwarded(obj)) + obj->zone()->removeUniqueId(obj); + else + MOZ_ASSERT(Forwarded(obj)->zone()->hasUniqueId(Forwarded(obj))); + } + cellsWithUid_.clear(); + + runSweepActions(); + sweepDictionaryModeObjects(); + +#ifdef JS_GC_ZEAL + /* Poison the nursery contents so touching a freed object will crash. */ + for (unsigned i = 0; i < numChunks(); i++) + chunk(i).poisonAndInit(runtime(), JS_SWEPT_NURSERY_PATTERN); + + if (runtime()->hasZealMode(ZealMode::GenerationalGC)) { + /* Only reset the alloc point when we are close to the end. */ + if (currentChunk_ + 1 == numChunks()) + setCurrentChunk(0); + } else +#endif + { +#ifdef JS_CRASH_DIAGNOSTICS + for (unsigned i = 0; i < numChunks(); ++i) + chunk(i).poisonAndInit(runtime(), JS_SWEPT_NURSERY_PATTERN); +#endif + setCurrentChunk(0); + } + + /* Set current start position for isEmpty checks. */ + setStartPosition(); + MemProfiler::SweepNursery(runtime()); +} + +size_t +js::Nursery::spaceToEnd() const +{ + unsigned lastChunk = numChunks() - 1; + + MOZ_ASSERT(lastChunk >= currentStartChunk_); + MOZ_ASSERT(currentStartPosition_ - chunk(currentStartChunk_).start() <= NurseryChunkUsableSize); + + size_t bytes = (chunk(currentStartChunk_).end() - currentStartPosition_) + + ((lastChunk - currentStartChunk_) * NurseryChunkUsableSize); + + MOZ_ASSERT(bytes <= numChunks() * NurseryChunkUsableSize); + + return bytes; +} + +MOZ_ALWAYS_INLINE void +js::Nursery::setCurrentChunk(unsigned chunkno) +{ + MOZ_ASSERT(chunkno < maxChunks()); + MOZ_ASSERT(chunkno < numChunks()); + currentChunk_ = chunkno; + position_ = chunk(chunkno).start(); + currentEnd_ = chunk(chunkno).end(); + chunk(chunkno).poisonAndInit(runtime(), JS_FRESH_NURSERY_PATTERN); +} + +MOZ_ALWAYS_INLINE void +js::Nursery::setStartPosition() +{ + currentStartChunk_ = currentChunk_; + currentStartPosition_ = position(); +} + +void +js::Nursery::maybeResizeNursery(JS::gcreason::Reason reason, double promotionRate) +{ + static const double GrowThreshold = 0.05; + static const double ShrinkThreshold = 0.01; + + // Shrink the nursery to its minimum size of we ran out of memory or + // received a memory pressure event. + if (gc::IsOOMReason(reason)) { + minimizeAllocableSpace(); + return; + } + + if (promotionRate > GrowThreshold) + growAllocableSpace(); + else if (promotionRate < ShrinkThreshold && previousPromotionRate_ < ShrinkThreshold) + shrinkAllocableSpace(); + + previousPromotionRate_ = promotionRate; +} + +void +js::Nursery::growAllocableSpace() +{ + updateNumChunks(Min(numChunks() * 2, maxNurseryChunks_)); +} + +void +js::Nursery::shrinkAllocableSpace() +{ +#ifdef JS_GC_ZEAL + if (runtime()->hasZealMode(ZealMode::GenerationalGC)) + return; +#endif + updateNumChunks(Max(numChunks() - 1, 1u)); +} + +void +js::Nursery::minimizeAllocableSpace() +{ +#ifdef JS_GC_ZEAL + if (runtime()->hasZealMode(ZealMode::GenerationalGC)) + return; +#endif + updateNumChunks(1); +} + +void +js::Nursery::updateNumChunks(unsigned newCount) +{ + if (numChunks() != newCount) { + AutoMaybeStartBackgroundAllocation maybeBgAlloc; + AutoLockGC lock(runtime()); + updateNumChunksLocked(newCount, maybeBgAlloc, lock); + } +} + +void +js::Nursery::updateNumChunksLocked(unsigned newCount, + AutoMaybeStartBackgroundAllocation& maybeBgAlloc, + AutoLockGC& lock) +{ + // The GC nursery is an optimization and so if we fail to allocate nursery + // chunks we do not report an error. + + unsigned priorCount = numChunks(); + MOZ_ASSERT(priorCount != newCount); + + if (newCount < priorCount) { + // Shrink the nursery and free unused chunks. + for (unsigned i = newCount; i < priorCount; i++) + runtime()->gc.recycleChunk(chunk(i).toChunk(runtime()), lock); + chunks_.shrinkTo(newCount); + return; + } + + // Grow the nursery and allocate new chunks. + if (!chunks_.resize(newCount)) + return; + + for (unsigned i = priorCount; i < newCount; i++) { + auto newChunk = runtime()->gc.getOrAllocChunk(lock, maybeBgAlloc); + if (!newChunk) { + chunks_.shrinkTo(i); + return; + } + + chunks_[i] = NurseryChunk::fromChunk(newChunk); + chunk(i).poisonAndInit(runtime(), JS_FRESH_NURSERY_PATTERN); + } +} + +void +js::Nursery::queueSweepAction(SweepThunk thunk, void* data) +{ + static_assert(sizeof(SweepAction) % CellSize == 0, + "SweepAction size must be a multiple of cell size"); + + MOZ_ASSERT(isEnabled()); + + AutoEnterOOMUnsafeRegion oomUnsafe; + auto action = reinterpret_cast<SweepAction*>(allocate(sizeof(SweepAction))); + if (!action) + oomUnsafe.crash("Nursery::queueSweepAction"); + + new (action) SweepAction(thunk, data, sweepActions_); + sweepActions_ = action; +} + +void +js::Nursery::runSweepActions() +{ + // The hazard analysis doesn't know whether the thunks can GC. + JS::AutoSuppressGCAnalysis nogc; + + AutoSetThreadIsSweeping threadIsSweeping; + for (auto action = sweepActions_; action; action = action->next) + action->thunk(action->data); + sweepActions_ = nullptr; +} + +bool +js::Nursery::queueDictionaryModeObjectToSweep(NativeObject* obj) +{ + MOZ_ASSERT(IsInsideNursery(obj)); + return dictionaryModeObjects_.append(obj); +} + +void +js::Nursery::sweepDictionaryModeObjects() +{ + for (auto obj : dictionaryModeObjects_) { + if (!IsForwarded(obj)) + obj->sweepDictionaryListPointer(); + } + dictionaryModeObjects_.clear(); +} diff --git a/js/src/gc/Nursery.h b/js/src/gc/Nursery.h new file mode 100644 index 000000000..69fb66b7a --- /dev/null +++ b/js/src/gc/Nursery.h @@ -0,0 +1,471 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sw=4 et tw=78: + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_Nursery_h +#define gc_Nursery_h + +#include "mozilla/EnumeratedArray.h" + +#include "jsalloc.h" +#include "jspubtd.h" + +#include "ds/BitArray.h" +#include "gc/Heap.h" +#include "gc/Memory.h" +#include "js/Class.h" +#include "js/GCAPI.h" +#include "js/HashTable.h" +#include "js/HeapAPI.h" +#include "js/Value.h" +#include "js/Vector.h" +#include "vm/SharedMem.h" + +#define FOR_EACH_NURSERY_PROFILE_TIME(_) \ + /* Key Header text */ \ + _(Total, "total") \ + _(CancelIonCompilations, "canIon") \ + _(TraceValues, "mkVals") \ + _(TraceCells, "mkClls") \ + _(TraceSlots, "mkSlts") \ + _(TraceWholeCells, "mcWCll") \ + _(TraceGenericEntries, "mkGnrc") \ + _(CheckHashTables, "ckTbls") \ + _(MarkRuntime, "mkRntm") \ + _(MarkDebugger, "mkDbgr") \ + _(ClearNewObjectCache, "clrNOC") \ + _(CollectToFP, "collct") \ + _(ObjectsTenuredCallback, "tenCB") \ + _(SweepArrayBufferViewList, "swpABO") \ + _(UpdateJitActivations, "updtIn") \ + _(FreeMallocedBuffers, "frSlts") \ + _(ClearStoreBuffer, "clrSB") \ + _(Sweep, "sweep") \ + _(Resize, "resize") \ + _(Pretenure, "pretnr") + +namespace JS { +struct Zone; +} // namespace JS + +namespace js { + +class ObjectElements; +class NativeObject; +class Nursery; +class HeapSlot; + +void SetGCZeal(JSRuntime*, uint8_t, uint32_t); + +namespace gc { +class AutoMaybeStartBackgroundAllocation; +struct Cell; +class MinorCollectionTracer; +class RelocationOverlay; +struct TenureCountCache; +} /* namespace gc */ + +namespace jit { +class MacroAssembler; +} // namespace jit + +class TenuringTracer : public JSTracer +{ + friend class Nursery; + Nursery& nursery_; + + // Amount of data moved to the tenured generation during collection. + size_t tenuredSize; + + // This list is threaded through the Nursery using the space from already + // moved things. The list is used to fix up the moved things and to find + // things held live by intra-Nursery pointers. + gc::RelocationOverlay* head; + gc::RelocationOverlay** tail; + + TenuringTracer(JSRuntime* rt, Nursery* nursery); + + public: + const Nursery& nursery() const { return nursery_; } + + // Returns true if the pointer was updated. + template <typename T> void traverse(T** thingp); + template <typename T> void traverse(T* thingp); + + void insertIntoFixupList(gc::RelocationOverlay* entry); + + // The store buffers need to be able to call these directly. + void traceObject(JSObject* src); + void traceObjectSlots(NativeObject* nobj, uint32_t start, uint32_t length); + void traceSlots(JS::Value* vp, uint32_t nslots) { traceSlots(vp, vp + nslots); } + + private: + Nursery& nursery() { return nursery_; } + + JSObject* moveToTenured(JSObject* src); + size_t moveObjectToTenured(JSObject* dst, JSObject* src, gc::AllocKind dstKind); + size_t moveElementsToTenured(NativeObject* dst, NativeObject* src, gc::AllocKind dstKind); + size_t moveSlotsToTenured(NativeObject* dst, NativeObject* src, gc::AllocKind dstKind); + + void traceSlots(JS::Value* vp, JS::Value* end); +}; + +/* + * Classes with JSCLASS_SKIP_NURSERY_FINALIZE or Wrapper classes with + * CROSS_COMPARTMENT flags will not have their finalizer called if they are + * nursery allocated and not promoted to the tenured heap. The finalizers for + * these classes must do nothing except free data which was allocated via + * Nursery::allocateBuffer. + */ +inline bool +CanNurseryAllocateFinalizedClass(const js::Class* const clasp) +{ + MOZ_ASSERT(clasp->hasFinalize()); + return clasp->flags & JSCLASS_SKIP_NURSERY_FINALIZE; +} + +class Nursery +{ + public: + static const size_t Alignment = gc::ChunkSize; + static const size_t ChunkShift = gc::ChunkShift; + + explicit Nursery(JSRuntime* rt); + ~Nursery(); + + MOZ_MUST_USE bool init(uint32_t maxNurseryBytes, AutoLockGC& lock); + + unsigned maxChunks() const { return maxNurseryChunks_; } + unsigned numChunks() const { return chunks_.length(); } + + bool exists() const { return maxChunks() != 0; } + size_t nurserySize() const { return maxChunks() << ChunkShift; } + + void enable(); + void disable(); + bool isEnabled() const { return numChunks() != 0; } + + /* Return true if no allocations have been made since the last collection. */ + bool isEmpty() const; + + /* + * Check whether an arbitrary pointer is within the nursery. This is + * slower than IsInsideNursery(Cell*), but works on all types of pointers. + */ + MOZ_ALWAYS_INLINE bool isInside(gc::Cell* cellp) const = delete; + MOZ_ALWAYS_INLINE bool isInside(const void* p) const { + for (auto chunk : chunks_) { + if (uintptr_t(p) - chunk->start() < gc::ChunkSize) + return true; + } + return false; + } + template<typename T> + bool isInside(const SharedMem<T>& p) const { + return isInside(p.unwrap(/*safe - used for value in comparison above*/)); + } + + /* + * Allocate and return a pointer to a new GC object with its |slots| + * pointer pre-filled. Returns nullptr if the Nursery is full. + */ + JSObject* allocateObject(JSContext* cx, size_t size, size_t numDynamic, const js::Class* clasp); + + /* Allocate a buffer for a given zone, using the nursery if possible. */ + void* allocateBuffer(JS::Zone* zone, size_t nbytes); + + /* + * Allocate a buffer for a given object, using the nursery if possible and + * obj is in the nursery. + */ + void* allocateBuffer(JSObject* obj, size_t nbytes); + + /* Resize an existing object buffer. */ + void* reallocateBuffer(JSObject* obj, void* oldBuffer, + size_t oldBytes, size_t newBytes); + + /* Free an object buffer. */ + void freeBuffer(void* buffer); + + /* The maximum number of bytes allowed to reside in nursery buffers. */ + static const size_t MaxNurseryBufferSize = 1024; + + /* Do a minor collection. */ + void collect(JSRuntime* rt, JS::gcreason::Reason reason); + + /* + * Check if the thing at |*ref| in the Nursery has been forwarded. If so, + * sets |*ref| to the new location of the object and returns true. Otherwise + * returns false and leaves |*ref| unset. + */ + MOZ_ALWAYS_INLINE MOZ_MUST_USE bool getForwardedPointer(JSObject** ref) const; + + /* Forward a slots/elements pointer stored in an Ion frame. */ + void forwardBufferPointer(HeapSlot** pSlotsElems); + + void maybeSetForwardingPointer(JSTracer* trc, void* oldData, void* newData, bool direct) { + if (trc->isTenuringTracer() && isInside(oldData)) + setForwardingPointer(oldData, newData, direct); + } + + /* Mark a malloced buffer as no longer needing to be freed. */ + void removeMallocedBuffer(void* buffer) { + mallocedBuffers.remove(buffer); + } + + void waitBackgroundFreeEnd(); + + MOZ_MUST_USE bool addedUniqueIdToCell(gc::Cell* cell) { + if (!IsInsideNursery(cell) || !isEnabled()) + return true; + MOZ_ASSERT(cellsWithUid_.initialized()); + MOZ_ASSERT(!cellsWithUid_.has(cell)); + return cellsWithUid_.put(cell); + } + + using SweepThunk = void (*)(void *data); + void queueSweepAction(SweepThunk thunk, void* data); + + MOZ_MUST_USE bool queueDictionaryModeObjectToSweep(NativeObject* obj); + + size_t sizeOfHeapCommitted() const { + return numChunks() * gc::ChunkSize; + } + size_t sizeOfMallocedBuffers(mozilla::MallocSizeOf mallocSizeOf) const { + size_t total = 0; + for (MallocedBuffersSet::Range r = mallocedBuffers.all(); !r.empty(); r.popFront()) + total += mallocSizeOf(r.front()); + total += mallocedBuffers.sizeOfExcludingThis(mallocSizeOf); + return total; + } + + // The number of bytes from the start position to the end of the nursery. + size_t spaceToEnd() const; + + // Free space remaining, not counting chunk trailers. + MOZ_ALWAYS_INLINE size_t freeSpace() const { + MOZ_ASSERT(currentEnd_ - position_ <= NurseryChunkUsableSize); + return (currentEnd_ - position_) + + (numChunks() - currentChunk_ - 1) * NurseryChunkUsableSize; + } + +#ifdef JS_GC_ZEAL + void enterZealMode(); + void leaveZealMode(); +#endif + + /* Print total profile times on shutdown. */ + void printTotalProfileTimes(); + + private: + /* The amount of space in the mapped nursery available to allocations. */ + static const size_t NurseryChunkUsableSize = gc::ChunkSize - sizeof(gc::ChunkTrailer); + + struct NurseryChunk { + char data[NurseryChunkUsableSize]; + gc::ChunkTrailer trailer; + static NurseryChunk* fromChunk(gc::Chunk* chunk); + void init(JSRuntime* rt); + void poisonAndInit(JSRuntime* rt, uint8_t poison); + uintptr_t start() const { return uintptr_t(&data); } + uintptr_t end() const { return uintptr_t(&trailer); } + gc::Chunk* toChunk(JSRuntime* rt); + }; + static_assert(sizeof(NurseryChunk) == gc::ChunkSize, + "Nursery chunk size must match gc::Chunk size."); + + /* + * The start and end pointers are stored under the runtime so that we can + * inline the isInsideNursery check into embedder code. Use the start() + * and heapEnd() functions to access these values. + */ + JSRuntime* runtime_; + + /* Vector of allocated chunks to allocate from. */ + Vector<NurseryChunk*, 0, SystemAllocPolicy> chunks_; + + /* Pointer to the first unallocated byte in the nursery. */ + uintptr_t position_; + + /* Pointer to the logical start of the Nursery. */ + unsigned currentStartChunk_; + uintptr_t currentStartPosition_; + + /* Pointer to the last byte of space in the current chunk. */ + uintptr_t currentEnd_; + + /* The index of the chunk that is currently being allocated from. */ + unsigned currentChunk_; + + /* Maximum number of chunks to allocate for the nursery. */ + unsigned maxNurseryChunks_; + + /* Promotion rate for the previous minor collection. */ + double previousPromotionRate_; + + /* Report minor collections taking at least this many us, if enabled. */ + int64_t profileThreshold_; + bool enableProfiling_; + + /* Report ObjectGroups with at lest this many instances tenured. */ + int64_t reportTenurings_; + + /* Profiling data. */ + + enum class ProfileKey + { +#define DEFINE_TIME_KEY(name, text) \ + name, + FOR_EACH_NURSERY_PROFILE_TIME(DEFINE_TIME_KEY) +#undef DEFINE_TIME_KEY + KeyCount + }; + + using ProfileTimes = mozilla::EnumeratedArray<ProfileKey, ProfileKey::KeyCount, int64_t>; + + ProfileTimes startTimes_; + ProfileTimes profileTimes_; + ProfileTimes totalTimes_; + uint64_t minorGcCount_; + + /* + * The set of externally malloced buffers potentially kept live by objects + * stored in the nursery. Any external buffers that do not belong to a + * tenured thing at the end of a minor GC must be freed. + */ + typedef HashSet<void*, PointerHasher<void*, 3>, SystemAllocPolicy> MallocedBuffersSet; + MallocedBuffersSet mallocedBuffers; + + /* A task structure used to free the malloced bufers on a background thread. */ + struct FreeMallocedBuffersTask; + FreeMallocedBuffersTask* freeMallocedBuffersTask; + + /* + * During a collection most hoisted slot and element buffers indicate their + * new location with a forwarding pointer at the base. This does not work + * for buffers whose length is less than pointer width, or when different + * buffers might overlap each other. For these, an entry in the following + * table is used. + */ + typedef HashMap<void*, void*, PointerHasher<void*, 1>, SystemAllocPolicy> ForwardedBufferMap; + ForwardedBufferMap forwardedBuffers; + + /* + * When we assign a unique id to cell in the nursery, that almost always + * means that the cell will be in a hash table, and thus, held live, + * automatically moving the uid from the nursery to its new home in + * tenured. It is possible, if rare, for an object that acquired a uid to + * be dead before the next collection, in which case we need to know to + * remove it when we sweep. + * + * Note: we store the pointers as Cell* here, resulting in an ugly cast in + * sweep. This is because this structure is used to help implement + * stable object hashing and we have to break the cycle somehow. + */ + using CellsWithUniqueIdSet = HashSet<gc::Cell*, PointerHasher<gc::Cell*, 3>, SystemAllocPolicy>; + CellsWithUniqueIdSet cellsWithUid_; + + struct SweepAction; + SweepAction* sweepActions_; + + using NativeObjectVector = Vector<NativeObject*, 0, SystemAllocPolicy>; + NativeObjectVector dictionaryModeObjects_; + +#ifdef JS_GC_ZEAL + struct Canary; + Canary* lastCanary_; +#endif + + NurseryChunk* allocChunk(); + + NurseryChunk& chunk(unsigned index) const { + return *chunks_[index]; + } + + void setCurrentChunk(unsigned chunkno); + void setStartPosition(); + + void updateNumChunks(unsigned newCount); + void updateNumChunksLocked(unsigned newCount, + gc::AutoMaybeStartBackgroundAllocation& maybeBgAlloc, + AutoLockGC& lock); + + MOZ_ALWAYS_INLINE uintptr_t allocationEnd() const { + MOZ_ASSERT(numChunks() > 0); + return chunks_.back()->end(); + } + + MOZ_ALWAYS_INLINE uintptr_t currentEnd() const { + MOZ_ASSERT(runtime_); + MOZ_ASSERT(currentEnd_ == chunk(currentChunk_).end()); + return currentEnd_; + } + void* addressOfCurrentEnd() const { + MOZ_ASSERT(runtime_); + return (void*)¤tEnd_; + } + + uintptr_t position() const { return position_; } + void* addressOfPosition() const { return (void*)&position_; } + + JSRuntime* runtime() const { return runtime_; } + + /* Allocates a new GC thing from the tenured generation during minor GC. */ + gc::TenuredCell* allocateFromTenured(JS::Zone* zone, gc::AllocKind thingKind); + + /* Common internal allocator function. */ + void* allocate(size_t size); + + double doCollection(JSRuntime* rt, JS::gcreason::Reason reason, + gc::TenureCountCache& tenureCounts); + + /* + * Move the object at |src| in the Nursery to an already-allocated cell + * |dst| in Tenured. + */ + void collectToFixedPoint(TenuringTracer& trc, gc::TenureCountCache& tenureCounts); + + /* Handle relocation of slots/elements pointers stored in Ion frames. */ + void setForwardingPointer(void* oldData, void* newData, bool direct); + + void setSlotsForwardingPointer(HeapSlot* oldSlots, HeapSlot* newSlots, uint32_t nslots); + void setElementsForwardingPointer(ObjectElements* oldHeader, ObjectElements* newHeader, + uint32_t nelems); + + /* Free malloced pointers owned by freed things in the nursery. */ + void freeMallocedBuffers(); + + /* + * Frees all non-live nursery-allocated things at the end of a minor + * collection. + */ + void sweep(); + + void runSweepActions(); + void sweepDictionaryModeObjects(); + + /* Change the allocable space provided by the nursery. */ + void maybeResizeNursery(JS::gcreason::Reason reason, double promotionRate); + void growAllocableSpace(); + void shrinkAllocableSpace(); + void minimizeAllocableSpace(); + + /* Profile recording and printing. */ + void startProfile(ProfileKey key); + void endProfile(ProfileKey key); + void maybeStartProfile(ProfileKey key); + void maybeEndProfile(ProfileKey key); + static void printProfileHeader(); + static void printProfileTimes(const ProfileTimes& times); + + friend class TenuringTracer; + friend class gc::MinorCollectionTracer; + friend class jit::MacroAssembler; +}; + +} /* namespace js */ + +#endif /* gc_Nursery_h */ diff --git a/js/src/gc/NurseryAwareHashMap.h b/js/src/gc/NurseryAwareHashMap.h new file mode 100644 index 000000000..5ade2e341 --- /dev/null +++ b/js/src/gc/NurseryAwareHashMap.h @@ -0,0 +1,178 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_NurseryAwareHashMap_h +#define gc_NurseryAwareHashMap_h + +namespace js { + +namespace detail { +// This class only handles the incremental case and does not deal with nursery +// pointers. The only users should be for NurseryAwareHashMap; it is defined +// externally because we need a GCPolicy for its use in the contained map. +template <typename T> +class UnsafeBareReadBarriered : public ReadBarrieredBase<T> +{ + public: + UnsafeBareReadBarriered() : ReadBarrieredBase<T>(JS::GCPolicy<T>::initial()) {} + MOZ_IMPLICIT UnsafeBareReadBarriered(const T& v) : ReadBarrieredBase<T>(v) {} + explicit UnsafeBareReadBarriered(const UnsafeBareReadBarriered& v) : ReadBarrieredBase<T>(v) {} + UnsafeBareReadBarriered(UnsafeBareReadBarriered&& v) + : ReadBarrieredBase<T>(mozilla::Move(v)) + {} + + UnsafeBareReadBarriered& operator=(const UnsafeBareReadBarriered& v) { + this->value = v.value; + return *this; + } + + UnsafeBareReadBarriered& operator=(const T& v) { + this->value = v; + return *this; + } + + const T get() const { + if (!InternalBarrierMethods<T>::isMarkable(this->value)) + return JS::GCPolicy<T>::initial(); + this->read(); + return this->value; + } + + explicit operator bool() const { + return bool(this->value); + } + + const T unbarrieredGet() const { return this->value; } + T* unsafeGet() { return &this->value; } + T const* unsafeGet() const { return &this->value; } +}; +} // namespace detail + +// The "nursery aware" hash map is a special case of GCHashMap that is able to +// treat nursery allocated members weakly during a minor GC: e.g. it allows for +// nursery allocated objects to be collected during nursery GC where a normal +// hash table treats such edges strongly. +// +// Doing this requires some strong constraints on what can be stored in this +// table and how it can be accessed. At the moment, this table assumes that +// all values contain a strong reference to the key. It also requires the +// policy to contain an |isTenured| and |needsSweep| members, which is fairly +// non-standard. This limits its usefulness to the CrossCompartmentMap at the +// moment, but might serve as a useful base for other tables in future. +template <typename Key, + typename Value, + typename HashPolicy = DefaultHasher<Key>, + typename AllocPolicy = TempAllocPolicy> +class NurseryAwareHashMap +{ + using BarrieredValue = detail::UnsafeBareReadBarriered<Value>; + using MapType = GCRekeyableHashMap<Key, BarrieredValue, HashPolicy, AllocPolicy>; + MapType map; + + // Keep a list of all keys for which JS::GCPolicy<Key>::isTenured is false. + // This lets us avoid a full traveral of the map on each minor GC, keeping + // the minor GC times proportional to the nursery heap size. + Vector<Key, 0, AllocPolicy> nurseryEntries; + + public: + using Lookup = typename MapType::Lookup; + using Ptr = typename MapType::Ptr; + using Range = typename MapType::Range; + + explicit NurseryAwareHashMap(AllocPolicy a = AllocPolicy()) : map(a) {} + + MOZ_MUST_USE bool init(uint32_t len = 16) { return map.init(len); } + + bool empty() const { return map.empty(); } + Ptr lookup(const Lookup& l) const { return map.lookup(l); } + void remove(Ptr p) { map.remove(p); } + Range all() const { return map.all(); } + struct Enum : public MapType::Enum { + explicit Enum(NurseryAwareHashMap& namap) : MapType::Enum(namap.map) {} + }; + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const { + return map.sizeOfExcludingThis(mallocSizeOf); + } + size_t sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const { + return map.sizeOfIncludingThis(mallocSizeOf); + } + + MOZ_MUST_USE bool put(const Key& k, const Value& v) { + auto p = map.lookupForAdd(k); + if (p) { + if (!JS::GCPolicy<Key>::isTenured(k) || !JS::GCPolicy<Value>::isTenured(v)) { + if (!nurseryEntries.append(k)) + return false; + } + p->value() = v; + return true; + } + + bool ok = map.add(p, k, v); + if (!ok) + return false; + + if (!JS::GCPolicy<Key>::isTenured(k) || !JS::GCPolicy<Value>::isTenured(v)) { + if (!nurseryEntries.append(k)) { + map.remove(k); + return false; + } + } + + return true; + } + + void sweepAfterMinorGC(JSTracer* trc) { + for (auto& key : nurseryEntries) { + auto p = map.lookup(key); + if (!p) + continue; + + // Drop the entry if the value is not marked. + if (JS::GCPolicy<BarrieredValue>::needsSweep(&p->value())) { + map.remove(key); + continue; + } + + // Update and relocate the key, if the value is still needed. + // + // Note that this currently assumes that all Value will contain a + // strong reference to Key, as per its use as the + // CrossCompartmentWrapperMap. We may need to make the following + // behavior more dynamic if we use this map in other nursery-aware + // contexts. + Key copy(key); + mozilla::DebugOnly<bool> sweepKey = JS::GCPolicy<Key>::needsSweep(©); + MOZ_ASSERT(!sweepKey); + map.rekeyIfMoved(key, copy); + } + nurseryEntries.clear(); + } + + void sweep() { + MOZ_ASSERT(nurseryEntries.empty()); + map.sweep(); + } +}; + +} // namespace js + +namespace JS { +template <typename T> +struct GCPolicy<js::detail::UnsafeBareReadBarriered<T>> +{ + static void trace(JSTracer* trc, js::detail::UnsafeBareReadBarriered<T>* thingp, + const char* name) + { + js::TraceEdge(trc, thingp, name); + } + static bool needsSweep(js::detail::UnsafeBareReadBarriered<T>* thingp) { + return js::gc::IsAboutToBeFinalized(thingp); + } +}; +} // namespace JS + +#endif // gc_NurseryAwareHashMap_h diff --git a/js/src/gc/Policy.h b/js/src/gc/Policy.h new file mode 100644 index 000000000..74b34d9c8 --- /dev/null +++ b/js/src/gc/Policy.h @@ -0,0 +1,159 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/* JS Garbage Collector. */ + +#ifndef gc_Policy_h +#define gc_Policy_h + +#include "mozilla/TypeTraits.h" +#include "gc/Barrier.h" +#include "gc/Marking.h" +#include "js/GCPolicyAPI.h" + +// Forward declare the types we're defining policies for. This file is +// included in all places that define GC things, so the real definitions +// will be available when we do template expansion, allowing for use of +// static members in the underlying types. We cannot, however, use +// static_assert to verify relations between types. +class JSLinearString; +namespace js { +class AccessorShape; +class ArgumentsObject; +class ArrayBufferObject; +class ArrayBufferObjectMaybeShared; +class ArrayBufferViewObject; +class ArrayObject; +class BaseShape; +class DebugEnvironmentProxy; +class DebuggerFrame; +class ExportEntryObject; +class EnvironmentObject; +class GlobalObject; +class ImportEntryObject; +class LazyScript; +class LexicalEnvironmentObject; +class ModuleEnvironmentObject; +class ModuleNamespaceObject; +class ModuleObject; +class NativeObject; +class ObjectGroup; +class PlainObject; +class PropertyName; +class RegExpObject; +class SavedFrame; +class Scope; +class EnvironmentObject; +class ScriptSourceObject; +class Shape; +class SharedArrayBufferObject; +class StructTypeDescr; +class UnownedBaseShape; +class WasmMemoryObject; +namespace jit { +class JitCode; +} // namespace jit +} // namespace js + +// Expand the given macro D for each valid GC reference type. +#define FOR_EACH_INTERNAL_GC_POINTER_TYPE(D) \ + D(JSFlatString*) \ + D(JSLinearString*) \ + D(js::AccessorShape*) \ + D(js::ArgumentsObject*) \ + D(js::ArrayBufferObject*) \ + D(js::ArrayBufferObjectMaybeShared*) \ + D(js::ArrayBufferViewObject*) \ + D(js::ArrayObject*) \ + D(js::BaseShape*) \ + D(js::DebugEnvironmentProxy*) \ + D(js::DebuggerFrame*) \ + D(js::ExportEntryObject*) \ + D(js::EnvironmentObject*) \ + D(js::GlobalObject*) \ + D(js::ImportEntryObject*) \ + D(js::LazyScript*) \ + D(js::LexicalEnvironmentObject*) \ + D(js::ModuleEnvironmentObject*) \ + D(js::ModuleNamespaceObject*) \ + D(js::ModuleObject*) \ + D(js::NativeObject*) \ + D(js::ObjectGroup*) \ + D(js::PlainObject*) \ + D(js::PropertyName*) \ + D(js::RegExpObject*) \ + D(js::SavedFrame*) \ + D(js::Scope*) \ + D(js::ScriptSourceObject*) \ + D(js::Shape*) \ + D(js::SharedArrayBufferObject*) \ + D(js::StructTypeDescr*) \ + D(js::UnownedBaseShape*) \ + D(js::WasmInstanceObject*) \ + D(js::WasmMemoryObject*) \ + D(js::WasmTableObject*) \ + D(js::jit::JitCode*) + +// Expand the given macro D for each internal tagged GC pointer type. +#define FOR_EACH_INTERNAL_TAGGED_GC_POINTER_TYPE(D) \ + D(js::TaggedProto) + +// Expand the macro D for every GC reference type that we know about. +#define FOR_EACH_GC_POINTER_TYPE(D) \ + FOR_EACH_PUBLIC_GC_POINTER_TYPE(D) \ + FOR_EACH_PUBLIC_TAGGED_GC_POINTER_TYPE(D) \ + FOR_EACH_INTERNAL_GC_POINTER_TYPE(D) \ + FOR_EACH_INTERNAL_TAGGED_GC_POINTER_TYPE(D) + +namespace js { + +// Define the GCPolicy for all internal pointers. +template <typename T> +struct InternalGCPointerPolicy { + using Type = typename mozilla::RemovePointer<T>::Type; + static T initial() { return nullptr; } + static void preBarrier(T v) { Type::writeBarrierPre(v); } + static void postBarrier(T* vp, T prev, T next) { Type::writeBarrierPost(vp, prev, next); } + static void readBarrier(T v) { Type::readBarrier(v); } + static void trace(JSTracer* trc, T* vp, const char* name) { + TraceManuallyBarrieredEdge(trc, vp, name); + } +}; + +} // namespace js + +namespace JS { + +#define DEFINE_INTERNAL_GC_POLICY(type) \ + template <> struct GCPolicy<type> : public js::InternalGCPointerPolicy<type> {}; +FOR_EACH_INTERNAL_GC_POINTER_TYPE(DEFINE_INTERNAL_GC_POLICY) +#undef DEFINE_INTERNAL_GC_POLICY + +template <typename T> +struct GCPolicy<js::HeapPtr<T>> +{ + static void trace(JSTracer* trc, js::HeapPtr<T>* thingp, const char* name) { + js::TraceEdge(trc, thingp, name); + } + static bool needsSweep(js::HeapPtr<T>* thingp) { + return js::gc::IsAboutToBeFinalized(thingp); + } +}; + +template <typename T> +struct GCPolicy<js::ReadBarriered<T>> +{ + static void trace(JSTracer* trc, js::ReadBarriered<T>* thingp, const char* name) { + js::TraceEdge(trc, thingp, name); + } + static bool needsSweep(js::ReadBarriered<T>* thingp) { + return js::gc::IsAboutToBeFinalized(thingp); + } +}; + +} // namespace JS + +#endif // gc_Policy_h diff --git a/js/src/gc/RootMarking.cpp b/js/src/gc/RootMarking.cpp new file mode 100644 index 000000000..93264084b --- /dev/null +++ b/js/src/gc/RootMarking.cpp @@ -0,0 +1,543 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/ArrayUtils.h" + +#ifdef MOZ_VALGRIND +# include <valgrind/memcheck.h> +#endif + +#include "jscntxt.h" +#include "jsgc.h" +#include "jsprf.h" +#include "jstypes.h" +#include "jswatchpoint.h" + +#include "builtin/MapObject.h" +#include "frontend/BytecodeCompiler.h" +#include "gc/GCInternals.h" +#include "gc/Marking.h" +#include "jit/MacroAssembler.h" +#include "js/HashTable.h" +#include "vm/Debugger.h" +#include "vm/JSONParser.h" + +#include "jsgcinlines.h" +#include "jsobjinlines.h" + +using namespace js; +using namespace js::gc; + +using mozilla::ArrayEnd; + +using JS::AutoGCRooter; + +typedef RootedValueMap::Range RootRange; +typedef RootedValueMap::Entry RootEntry; +typedef RootedValueMap::Enum RootEnum; + +template <typename T> +using TraceFunction = void (*)(JSTracer* trc, T* ref, const char* name); + +// For more detail see JS::Rooted::ptr and js::DispatchWrapper. +// +// The JS::RootKind::Traceable list contains a bunch of totally disparate +// types, but the instantiations of DispatchWrapper below need /something/ in +// the type field. We use the following type as a compatible stand-in. No +// actual methods from ConcreteTraceable type are actually used at runtime -- +// the real trace function has been stored inline in the DispatchWrapper. +struct ConcreteTraceable { + ConcreteTraceable() { MOZ_CRASH("instantiation of ConcreteTraceable"); } + void trace(JSTracer*) {} +}; + +template <typename T, TraceFunction<T> TraceFn = TraceNullableRoot> +static inline void +MarkExactStackRootList(JSTracer* trc, JS::Rooted<void*>* rooter, const char* name) +{ + while (rooter) { + T* addr = reinterpret_cast<JS::Rooted<T>*>(rooter)->address(); + TraceFn(trc, addr, name); + rooter = rooter->previous(); + } +} + +static inline void +TraceStackRoots(JSTracer* trc, RootedListHeads& stackRoots) +{ +#define MARK_ROOTS(name, type, _) \ + MarkExactStackRootList<type*>(trc, stackRoots[JS::RootKind::name], "exact-" #name); +JS_FOR_EACH_TRACEKIND(MARK_ROOTS) +#undef MARK_ROOTS + MarkExactStackRootList<jsid>(trc, stackRoots[JS::RootKind::Id], "exact-id"); + MarkExactStackRootList<Value>(trc, stackRoots[JS::RootKind::Value], "exact-value"); + MarkExactStackRootList<ConcreteTraceable, + js::DispatchWrapper<ConcreteTraceable>::TraceWrapped>( + trc, stackRoots[JS::RootKind::Traceable], "Traceable"); +} + +void +js::RootLists::traceStackRoots(JSTracer* trc) +{ + TraceStackRoots(trc, stackRoots_); +} + +static void +MarkExactStackRoots(JSRuntime* rt, JSTracer* trc) +{ + for (ZonesIter zone(rt, SkipAtoms); !zone.done(); zone.next()) + TraceStackRoots(trc, zone->stackRoots_); + rt->contextFromMainThread()->roots.traceStackRoots(trc); +} + +template <typename T, TraceFunction<T> TraceFn = TraceNullableRoot> +static inline void +MarkPersistentRootedList(JSTracer* trc, mozilla::LinkedList<PersistentRooted<void*>>& list, + const char* name) +{ + for (PersistentRooted<void*>* r : list) + TraceFn(trc, reinterpret_cast<PersistentRooted<T>*>(r)->address(), name); +} + +void +js::RootLists::tracePersistentRoots(JSTracer* trc) +{ +#define MARK_ROOTS(name, type, _) \ + MarkPersistentRootedList<type*>(trc, heapRoots_[JS::RootKind::name], "persistent-" #name); +JS_FOR_EACH_TRACEKIND(MARK_ROOTS) +#undef MARK_ROOTS + MarkPersistentRootedList<jsid>(trc, heapRoots_[JS::RootKind::Id], "persistent-id"); + MarkPersistentRootedList<Value>(trc, heapRoots_[JS::RootKind::Value], "persistent-value"); + MarkPersistentRootedList<ConcreteTraceable, + js::DispatchWrapper<ConcreteTraceable>::TraceWrapped>(trc, + heapRoots_[JS::RootKind::Traceable], "persistent-traceable"); +} + +static void +MarkPersistentRooted(JSRuntime* rt, JSTracer* trc) +{ + rt->contextFromMainThread()->roots.tracePersistentRoots(trc); +} + +template <typename T> +static void +FinishPersistentRootedChain(mozilla::LinkedList<PersistentRooted<void*>>& listArg) +{ + auto& list = reinterpret_cast<mozilla::LinkedList<PersistentRooted<T>>&>(listArg); + while (!list.isEmpty()) + list.getFirst()->reset(); +} + +void +js::RootLists::finishPersistentRoots() +{ +#define FINISH_ROOT_LIST(name, type, _) \ + FinishPersistentRootedChain<type*>(heapRoots_[JS::RootKind::name]); +JS_FOR_EACH_TRACEKIND(FINISH_ROOT_LIST) +#undef FINISH_ROOT_LIST + FinishPersistentRootedChain<jsid>(heapRoots_[JS::RootKind::Id]); + FinishPersistentRootedChain<Value>(heapRoots_[JS::RootKind::Value]); + + // Note that we do not finalize the Traceable list as we do not know how to + // safely clear memebers. We instead assert that none escape the RootLists. + // See the comment on RootLists::~RootLists for details. +} + +inline void +AutoGCRooter::trace(JSTracer* trc) +{ + switch (tag_) { + case PARSER: + frontend::MarkParser(trc, this); + return; + + case VALARRAY: { + /* + * We don't know the template size parameter, but we can safely treat it + * as an AutoValueArray<1> because the length is stored separately. + */ + AutoValueArray<1>* array = static_cast<AutoValueArray<1>*>(this); + TraceRootRange(trc, array->length(), array->begin(), "js::AutoValueArray"); + return; + } + + case IONMASM: { + static_cast<js::jit::MacroAssembler::AutoRooter*>(this)->masm()->trace(trc); + return; + } + + case WRAPPER: { + /* + * We need to use TraceManuallyBarrieredEdge here because we mark + * wrapper roots in every slice. This is because of some rule-breaking + * in RemapAllWrappersForObject; see comment there. + */ + TraceManuallyBarrieredEdge(trc, &static_cast<AutoWrapperRooter*>(this)->value.get(), + "JS::AutoWrapperRooter.value"); + return; + } + + case WRAPVECTOR: { + AutoWrapperVector::VectorImpl& vector = static_cast<AutoWrapperVector*>(this)->vector; + /* + * We need to use TraceManuallyBarrieredEdge here because we mark + * wrapper roots in every slice. This is because of some rule-breaking + * in RemapAllWrappersForObject; see comment there. + */ + for (WrapperValue* p = vector.begin(); p < vector.end(); p++) + TraceManuallyBarrieredEdge(trc, &p->get(), "js::AutoWrapperVector.vector"); + return; + } + + case CUSTOM: + static_cast<JS::CustomAutoRooter*>(this)->trace(trc); + return; + } + + MOZ_ASSERT(tag_ >= 0); + if (Value* vp = static_cast<AutoArrayRooter*>(this)->array) + TraceRootRange(trc, tag_, vp, "JS::AutoArrayRooter.array"); +} + +/* static */ void +AutoGCRooter::traceAll(JSTracer* trc) +{ + for (AutoGCRooter* gcr = trc->runtime()->contextFromMainThread()->roots.autoGCRooters_; gcr; gcr = gcr->down) + gcr->trace(trc); +} + +/* static */ void +AutoGCRooter::traceAllWrappers(JSTracer* trc) +{ + JSContext* cx = trc->runtime()->contextFromMainThread(); + + for (AutoGCRooter* gcr = cx->roots.autoGCRooters_; gcr; gcr = gcr->down) { + if (gcr->tag_ == WRAPVECTOR || gcr->tag_ == WRAPPER) + gcr->trace(trc); + } +} + +void +StackShape::trace(JSTracer* trc) +{ + if (base) + TraceRoot(trc, &base, "StackShape base"); + + TraceRoot(trc, (jsid*) &propid, "StackShape id"); + + if ((attrs & JSPROP_GETTER) && rawGetter) + TraceRoot(trc, (JSObject**)&rawGetter, "StackShape getter"); + + if ((attrs & JSPROP_SETTER) && rawSetter) + TraceRoot(trc, (JSObject**)&rawSetter, "StackShape setter"); +} + +void +PropertyDescriptor::trace(JSTracer* trc) +{ + if (obj) + TraceRoot(trc, &obj, "Descriptor::obj"); + TraceRoot(trc, &value, "Descriptor::value"); + if ((attrs & JSPROP_GETTER) && getter) { + JSObject* tmp = JS_FUNC_TO_DATA_PTR(JSObject*, getter); + TraceRoot(trc, &tmp, "Descriptor::get"); + getter = JS_DATA_TO_FUNC_PTR(JSGetterOp, tmp); + } + if ((attrs & JSPROP_SETTER) && setter) { + JSObject* tmp = JS_FUNC_TO_DATA_PTR(JSObject*, setter); + TraceRoot(trc, &tmp, "Descriptor::set"); + setter = JS_DATA_TO_FUNC_PTR(JSSetterOp, tmp); + } +} + +void +js::gc::GCRuntime::traceRuntimeForMajorGC(JSTracer* trc, AutoLockForExclusiveAccess& lock) +{ + // FinishRoots will have asserted that every root that we do not expect + // is gone, so we can simply skip traceRuntime here. + if (rt->isBeingDestroyed()) + return; + + gcstats::AutoPhase ap(stats, gcstats::PHASE_MARK_ROOTS); + if (rt->atomsCompartment(lock)->zone()->isCollecting()) + traceRuntimeAtoms(trc, lock); + JSCompartment::traceIncomingCrossCompartmentEdgesForZoneGC(trc); + traceRuntimeCommon(trc, MarkRuntime, lock); +} + +void +js::gc::GCRuntime::traceRuntimeForMinorGC(JSTracer* trc, AutoLockForExclusiveAccess& lock) +{ + // Note that we *must* trace the runtime during the SHUTDOWN_GC's minor GC + // despite having called FinishRoots already. This is because FinishRoots + // does not clear the crossCompartmentWrapper map. It cannot do this + // because Proxy's trace for CrossCompartmentWrappers asserts presence in + // the map. And we can reach its trace function despite having finished the + // roots via the edges stored by the pre-barrier verifier when we finish + // the verifier for the last time. + gcstats::AutoPhase ap(stats, gcstats::PHASE_MARK_ROOTS); + + // FIXME: As per bug 1298816 comment 12, we should be able to remove this. + jit::JitRuntime::MarkJitcodeGlobalTableUnconditionally(trc); + + traceRuntimeCommon(trc, TraceRuntime, lock); +} + +void +js::TraceRuntime(JSTracer* trc) +{ + MOZ_ASSERT(!trc->isMarkingTracer()); + + JSRuntime* rt = trc->runtime(); + rt->gc.evictNursery(); + AutoPrepareForTracing prep(rt->contextFromMainThread(), WithAtoms); + gcstats::AutoPhase ap(rt->gc.stats, gcstats::PHASE_TRACE_HEAP); + rt->gc.traceRuntime(trc, prep.session().lock); +} + +void +js::gc::GCRuntime::traceRuntime(JSTracer* trc, AutoLockForExclusiveAccess& lock) +{ + MOZ_ASSERT(!rt->isBeingDestroyed()); + + gcstats::AutoPhase ap(stats, gcstats::PHASE_MARK_ROOTS); + traceRuntimeAtoms(trc, lock); + traceRuntimeCommon(trc, TraceRuntime, lock); +} + +void +js::gc::GCRuntime::traceRuntimeAtoms(JSTracer* trc, AutoLockForExclusiveAccess& lock) +{ + gcstats::AutoPhase ap(stats, gcstats::PHASE_MARK_RUNTIME_DATA); + MarkPermanentAtoms(trc); + MarkAtoms(trc, lock); + MarkWellKnownSymbols(trc); + jit::JitRuntime::Mark(trc, lock); +} + +void +js::gc::GCRuntime::traceRuntimeCommon(JSTracer* trc, TraceOrMarkRuntime traceOrMark, + AutoLockForExclusiveAccess& lock) +{ + MOZ_ASSERT(!rt->mainThread.suppressGC); + + { + gcstats::AutoPhase ap(stats, gcstats::PHASE_MARK_STACK); + + // Trace active interpreter and JIT stack roots. + MarkInterpreterActivations(rt, trc); + jit::MarkJitActivations(rt, trc); + + // Trace legacy C stack roots. + AutoGCRooter::traceAll(trc); + + for (RootRange r = rootsHash.all(); !r.empty(); r.popFront()) { + const RootEntry& entry = r.front(); + TraceRoot(trc, entry.key(), entry.value()); + } + + // Trace C stack roots. + MarkExactStackRoots(rt, trc); + } + + // Trace runtime global roots. + MarkPersistentRooted(rt, trc); + + // Trace the self-hosting global compartment. + rt->markSelfHostingGlobal(trc); + + // Trace the shared Intl data. + rt->traceSharedIntlData(trc); + + // Trace anything in the single context. Note that this is actually the + // same struct as the JSRuntime, but is still split for historical reasons. + rt->contextFromMainThread()->mark(trc); + + // Trace all compartment roots, but not the compartment itself; it is + // marked via the parent pointer if traceRoots actually traces anything. + for (CompartmentsIter c(rt, SkipAtoms); !c.done(); c.next()) + c->traceRoots(trc, traceOrMark); + + // Trace SPS. + rt->spsProfiler.trace(trc); + + // Trace helper thread roots. + HelperThreadState().trace(trc); + + // Trace the embedding's black and gray roots. + if (!rt->isHeapMinorCollecting()) { + gcstats::AutoPhase ap(stats, gcstats::PHASE_MARK_EMBEDDING); + + /* + * The embedding can register additional roots here. + * + * We don't need to trace these in a minor GC because all pointers into + * the nursery should be in the store buffer, and we want to avoid the + * time taken to trace all these roots. + */ + for (size_t i = 0; i < blackRootTracers.length(); i++) { + const Callback<JSTraceDataOp>& e = blackRootTracers[i]; + (*e.op)(trc, e.data); + } + + /* During GC, we don't mark gray roots at this stage. */ + if (JSTraceDataOp op = grayRootTracer.op) { + if (traceOrMark == TraceRuntime) + (*op)(trc, grayRootTracer.data); + } + } +} + +#ifdef DEBUG +class AssertNoRootsTracer : public JS::CallbackTracer +{ + void onChild(const JS::GCCellPtr& thing) override { + MOZ_CRASH("There should not be any roots after finishRoots"); + } + + public: + AssertNoRootsTracer(JSRuntime* rt, WeakMapTraceKind weakTraceKind) + : JS::CallbackTracer(rt, weakTraceKind) + {} +}; +#endif // DEBUG + +void +js::gc::GCRuntime::finishRoots() +{ + rt->finishAtoms(); + + if (rootsHash.initialized()) + rootsHash.clear(); + + rt->contextFromMainThread()->roots.finishPersistentRoots(); + + rt->finishSelfHosting(); + + for (CompartmentsIter c(rt, SkipAtoms); !c.done(); c.next()) + c->finishRoots(); + +#ifdef DEBUG + // The nsWrapperCache may not be empty before our shutdown GC, so we have + // to skip that table when verifying that we are fully unrooted. + auto prior = grayRootTracer; + grayRootTracer = Callback<JSTraceDataOp>(nullptr, nullptr); + + AssertNoRootsTracer trc(rt, TraceWeakMapKeysValues); + AutoPrepareForTracing prep(rt->contextFromMainThread(), WithAtoms); + gcstats::AutoPhase ap(rt->gc.stats, gcstats::PHASE_TRACE_HEAP); + traceRuntime(&trc, prep.session().lock); + + // Restore the wrapper tracing so that we leak instead of leaving dangling + // pointers. + grayRootTracer = prior; +#endif // DEBUG +} + +// Append traced things to a buffer on the zone for use later in the GC. +// See the comment in GCRuntime.h above grayBufferState for details. +class BufferGrayRootsTracer : public JS::CallbackTracer +{ + // Set to false if we OOM while buffering gray roots. + bool bufferingGrayRootsFailed; + + void onChild(const JS::GCCellPtr& thing) override; + + public: + explicit BufferGrayRootsTracer(JSRuntime* rt) + : JS::CallbackTracer(rt), bufferingGrayRootsFailed(false) + {} + + bool failed() const { return bufferingGrayRootsFailed; } + +#ifdef DEBUG + TracerKind getTracerKind() const override { return TracerKind::GrayBuffering; } +#endif +}; + +#ifdef DEBUG +// Return true if this trace is happening on behalf of gray buffering during +// the marking phase of incremental GC. +bool +js::IsBufferGrayRootsTracer(JSTracer* trc) +{ + return trc->isCallbackTracer() && + trc->asCallbackTracer()->getTracerKind() == JS::CallbackTracer::TracerKind::GrayBuffering; +} +#endif + +void +js::gc::GCRuntime::bufferGrayRoots() +{ + // Precondition: the state has been reset to "unused" after the last GC + // and the zone's buffers have been cleared. + MOZ_ASSERT(grayBufferState == GrayBufferState::Unused); + for (GCZonesIter zone(rt); !zone.done(); zone.next()) + MOZ_ASSERT(zone->gcGrayRoots.empty()); + + + BufferGrayRootsTracer grayBufferer(rt); + if (JSTraceDataOp op = grayRootTracer.op) + (*op)(&grayBufferer, grayRootTracer.data); + + // Propagate the failure flag from the marker to the runtime. + if (grayBufferer.failed()) { + grayBufferState = GrayBufferState::Failed; + resetBufferedGrayRoots(); + } else { + grayBufferState = GrayBufferState::Okay; + } +} + +struct SetMaybeAliveFunctor { + template <typename T> void operator()(T* t) { SetMaybeAliveFlag(t); } +}; + +void +BufferGrayRootsTracer::onChild(const JS::GCCellPtr& thing) +{ + MOZ_ASSERT(runtime()->isHeapBusy()); + MOZ_RELEASE_ASSERT(thing); + // Check if |thing| is corrupt by calling a method that touches the heap. + MOZ_RELEASE_ASSERT(thing.asCell()->getTraceKind() <= JS::TraceKind::Null); + + if (bufferingGrayRootsFailed) + return; + + gc::TenuredCell* tenured = gc::TenuredCell::fromPointer(thing.asCell()); + + Zone* zone = tenured->zone(); + if (zone->isCollecting()) { + // See the comment on SetMaybeAliveFlag to see why we only do this for + // objects and scripts. We rely on gray root buffering for this to work, + // but we only need to worry about uncollected dead compartments during + // incremental GCs (when we do gray root buffering). + DispatchTyped(SetMaybeAliveFunctor(), thing); + + if (!zone->gcGrayRoots.append(tenured)) + bufferingGrayRootsFailed = true; + } +} + +void +GCRuntime::markBufferedGrayRoots(JS::Zone* zone) +{ + MOZ_ASSERT(grayBufferState == GrayBufferState::Okay); + MOZ_ASSERT(zone->isGCMarkingGray() || zone->isGCCompacting()); + + for (auto cell : zone->gcGrayRoots) + TraceManuallyBarrieredGenericPointerEdge(&marker, &cell, "buffered gray root"); +} + +void +GCRuntime::resetBufferedGrayRoots() const +{ + MOZ_ASSERT(grayBufferState != GrayBufferState::Okay, + "Do not clear the gray buffers unless we are Failed or becoming Unused"); + for (GCZonesIter zone(rt); !zone.done(); zone.next()) + zone->gcGrayRoots.clearAndFree(); +} + diff --git a/js/src/gc/Rooting.h b/js/src/gc/Rooting.h new file mode 100644 index 000000000..7a179aa28 --- /dev/null +++ b/js/src/gc/Rooting.h @@ -0,0 +1,82 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_Rooting_h +#define gc_Rooting_h + +#include "js/GCVector.h" +#include "js/RootingAPI.h" + +class JSAtom; +class JSLinearString; + +namespace js { + +class PropertyName; +class NativeObject; +class ArrayObject; +class GlobalObject; +class PlainObject; +class ScriptSourceObject; +class SavedFrame; +class Shape; +class ObjectGroup; +class DebuggerEnvironment; +class DebuggerFrame; +class DebuggerObject; +class Scope; + +// These are internal counterparts to the public types such as HandleObject. + +typedef JS::Handle<NativeObject*> HandleNativeObject; +typedef JS::Handle<Shape*> HandleShape; +typedef JS::Handle<ObjectGroup*> HandleObjectGroup; +typedef JS::Handle<JSAtom*> HandleAtom; +typedef JS::Handle<JSLinearString*> HandleLinearString; +typedef JS::Handle<PropertyName*> HandlePropertyName; +typedef JS::Handle<ArrayObject*> HandleArrayObject; +typedef JS::Handle<PlainObject*> HandlePlainObject; +typedef JS::Handle<SavedFrame*> HandleSavedFrame; +typedef JS::Handle<ScriptSourceObject*> HandleScriptSource; +typedef JS::Handle<DebuggerEnvironment*> HandleDebuggerEnvironment; +typedef JS::Handle<DebuggerFrame*> HandleDebuggerFrame; +typedef JS::Handle<DebuggerObject*> HandleDebuggerObject; +typedef JS::Handle<Scope*> HandleScope; + +typedef JS::MutableHandle<Shape*> MutableHandleShape; +typedef JS::MutableHandle<JSAtom*> MutableHandleAtom; +typedef JS::MutableHandle<NativeObject*> MutableHandleNativeObject; +typedef JS::MutableHandle<PlainObject*> MutableHandlePlainObject; +typedef JS::MutableHandle<SavedFrame*> MutableHandleSavedFrame; +typedef JS::MutableHandle<DebuggerEnvironment*> MutableHandleDebuggerEnvironment; +typedef JS::MutableHandle<DebuggerFrame*> MutableHandleDebuggerFrame; +typedef JS::MutableHandle<DebuggerObject*> MutableHandleDebuggerObject; +typedef JS::MutableHandle<Scope*> MutableHandleScope; + +typedef JS::Rooted<NativeObject*> RootedNativeObject; +typedef JS::Rooted<Shape*> RootedShape; +typedef JS::Rooted<ObjectGroup*> RootedObjectGroup; +typedef JS::Rooted<JSAtom*> RootedAtom; +typedef JS::Rooted<JSLinearString*> RootedLinearString; +typedef JS::Rooted<PropertyName*> RootedPropertyName; +typedef JS::Rooted<ArrayObject*> RootedArrayObject; +typedef JS::Rooted<GlobalObject*> RootedGlobalObject; +typedef JS::Rooted<PlainObject*> RootedPlainObject; +typedef JS::Rooted<SavedFrame*> RootedSavedFrame; +typedef JS::Rooted<ScriptSourceObject*> RootedScriptSource; +typedef JS::Rooted<DebuggerEnvironment*> RootedDebuggerEnvironment; +typedef JS::Rooted<DebuggerFrame*> RootedDebuggerFrame; +typedef JS::Rooted<DebuggerObject*> RootedDebuggerObject; +typedef JS::Rooted<Scope*> RootedScope; + +typedef JS::GCVector<JSFunction*> FunctionVector; +typedef JS::GCVector<PropertyName*> PropertyNameVector; +typedef JS::GCVector<Shape*> ShapeVector; +typedef JS::GCVector<JSString*> StringVector; + +} /* namespace js */ + +#endif /* gc_Rooting_h */ diff --git a/js/src/gc/Statistics.cpp b/js/src/gc/Statistics.cpp new file mode 100644 index 000000000..19f9986dd --- /dev/null +++ b/js/src/gc/Statistics.cpp @@ -0,0 +1,1383 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/Statistics.h" + +#include "mozilla/ArrayUtils.h" +#include "mozilla/DebugOnly.h" +#include "mozilla/IntegerRange.h" +#include "mozilla/PodOperations.h" +#include "mozilla/Sprintf.h" + +#include <ctype.h> +#include <stdarg.h> +#include <stdio.h> + +#include "jsprf.h" +#include "jsutil.h" + +#include "gc/Memory.h" +#include "vm/Debugger.h" +#include "vm/HelperThreads.h" +#include "vm/Runtime.h" +#include "vm/Time.h" + +using namespace js; +using namespace js::gc; +using namespace js::gcstats; + +using mozilla::DebugOnly; +using mozilla::MakeRange; +using mozilla::PodArrayZero; +using mozilla::PodZero; + +/* + * If this fails, then you can either delete this assertion and allow all + * larger-numbered reasons to pile up in the last telemetry bucket, or switch + * to GC_REASON_3 and bump the max value. + */ +JS_STATIC_ASSERT(JS::gcreason::NUM_TELEMETRY_REASONS >= JS::gcreason::NUM_REASONS); + +const char* +js::gcstats::ExplainInvocationKind(JSGCInvocationKind gckind) +{ + MOZ_ASSERT(gckind == GC_NORMAL || gckind == GC_SHRINK); + if (gckind == GC_NORMAL) + return "Normal"; + else + return "Shrinking"; +} + +JS_PUBLIC_API(const char*) +JS::gcreason::ExplainReason(JS::gcreason::Reason reason) +{ + switch (reason) { +#define SWITCH_REASON(name) \ + case JS::gcreason::name: \ + return #name; + GCREASONS(SWITCH_REASON) + + default: + MOZ_CRASH("bad GC reason"); +#undef SWITCH_REASON + } +} + +const char* +js::gcstats::ExplainAbortReason(gc::AbortReason reason) +{ + switch (reason) { +#define SWITCH_REASON(name) \ + case gc::AbortReason::name: \ + return #name; + GC_ABORT_REASONS(SWITCH_REASON) + + default: + MOZ_CRASH("bad GC abort reason"); +#undef SWITCH_REASON + } +} + +static double +t(int64_t t) +{ + return double(t) / PRMJ_USEC_PER_MSEC; +} + +struct PhaseInfo +{ + Phase index; + const char* name; + Phase parent; + const uint8_t telemetryBucket; +}; + +// The zeroth entry in the timing arrays is used for phases that have a +// unique lineage. +static const size_t PHASE_DAG_NONE = 0; + +// These are really just fields of PhaseInfo, but I have to initialize them +// programmatically, which prevents making phases[] const. (And marking these +// fields mutable does not work on Windows; the whole thing gets created in +// read-only memory anyway.) +struct ExtraPhaseInfo +{ + // Depth in the tree of each phase type + size_t depth; + + // Index into the set of parallel arrays of timing data, for parents with + // at least one multi-parented child + size_t dagSlot; +}; + +static const Phase PHASE_NO_PARENT = PHASE_LIMIT; + +struct DagChildEdge { + Phase parent; + Phase child; +} dagChildEdges[] = { + { PHASE_MARK, PHASE_MARK_ROOTS }, + { PHASE_MINOR_GC, PHASE_MARK_ROOTS }, + { PHASE_TRACE_HEAP, PHASE_MARK_ROOTS }, + { PHASE_EVICT_NURSERY, PHASE_MARK_ROOTS }, + { PHASE_COMPACT_UPDATE, PHASE_MARK_ROOTS } +}; + +/* + * Note that PHASE_MUTATOR, PHASE_GC_BEGIN, and PHASE_GC_END never have any + * child phases. If beginPhase is called while one of these is active, they + * will automatically be suspended and resumed when the phase stack is next + * empty. Timings for these phases are thus exclusive of any other phase. + */ + +static const PhaseInfo phases[] = { + { PHASE_MUTATOR, "Mutator Running", PHASE_NO_PARENT, 0 }, + { PHASE_GC_BEGIN, "Begin Callback", PHASE_NO_PARENT, 1 }, + { PHASE_WAIT_BACKGROUND_THREAD, "Wait Background Thread", PHASE_NO_PARENT, 2 }, + { PHASE_MARK_DISCARD_CODE, "Mark Discard Code", PHASE_NO_PARENT, 3 }, + { PHASE_RELAZIFY_FUNCTIONS, "Relazify Functions", PHASE_NO_PARENT, 4 }, + { PHASE_PURGE, "Purge", PHASE_NO_PARENT, 5 }, + { PHASE_MARK, "Mark", PHASE_NO_PARENT, 6 }, + { PHASE_UNMARK, "Unmark", PHASE_MARK, 7 }, + /* PHASE_MARK_ROOTS */ + { PHASE_MARK_DELAYED, "Mark Delayed", PHASE_MARK, 8 }, + { PHASE_SWEEP, "Sweep", PHASE_NO_PARENT, 9 }, + { PHASE_SWEEP_MARK, "Mark During Sweeping", PHASE_SWEEP, 10 }, + { PHASE_SWEEP_MARK_TYPES, "Mark Types During Sweeping", PHASE_SWEEP_MARK, 11 }, + { PHASE_SWEEP_MARK_INCOMING_BLACK, "Mark Incoming Black Pointers", PHASE_SWEEP_MARK, 12 }, + { PHASE_SWEEP_MARK_WEAK, "Mark Weak", PHASE_SWEEP_MARK, 13 }, + { PHASE_SWEEP_MARK_INCOMING_GRAY, "Mark Incoming Gray Pointers", PHASE_SWEEP_MARK, 14 }, + { PHASE_SWEEP_MARK_GRAY, "Mark Gray", PHASE_SWEEP_MARK, 15 }, + { PHASE_SWEEP_MARK_GRAY_WEAK, "Mark Gray and Weak", PHASE_SWEEP_MARK, 16 }, + { PHASE_FINALIZE_START, "Finalize Start Callbacks", PHASE_SWEEP, 17 }, + { PHASE_WEAK_ZONEGROUP_CALLBACK, "Per-Slice Weak Callback", PHASE_FINALIZE_START, 57 }, + { PHASE_WEAK_COMPARTMENT_CALLBACK, "Per-Compartment Weak Callback", PHASE_FINALIZE_START, 58 }, + { PHASE_SWEEP_ATOMS, "Sweep Atoms", PHASE_SWEEP, 18 }, + { PHASE_SWEEP_SYMBOL_REGISTRY, "Sweep Symbol Registry", PHASE_SWEEP, 19 }, + { PHASE_SWEEP_COMPARTMENTS, "Sweep Compartments", PHASE_SWEEP, 20 }, + { PHASE_SWEEP_DISCARD_CODE, "Sweep Discard Code", PHASE_SWEEP_COMPARTMENTS, 21 }, + { PHASE_SWEEP_INNER_VIEWS, "Sweep Inner Views", PHASE_SWEEP_COMPARTMENTS, 22 }, + { PHASE_SWEEP_CC_WRAPPER, "Sweep Cross Compartment Wrappers", PHASE_SWEEP_COMPARTMENTS, 23 }, + { PHASE_SWEEP_BASE_SHAPE, "Sweep Base Shapes", PHASE_SWEEP_COMPARTMENTS, 24 }, + { PHASE_SWEEP_INITIAL_SHAPE, "Sweep Initial Shapes", PHASE_SWEEP_COMPARTMENTS, 25 }, + { PHASE_SWEEP_TYPE_OBJECT, "Sweep Type Objects", PHASE_SWEEP_COMPARTMENTS, 26 }, + { PHASE_SWEEP_BREAKPOINT, "Sweep Breakpoints", PHASE_SWEEP_COMPARTMENTS, 27 }, + { PHASE_SWEEP_REGEXP, "Sweep Regexps", PHASE_SWEEP_COMPARTMENTS, 28 }, + { PHASE_SWEEP_MISC, "Sweep Miscellaneous", PHASE_SWEEP_COMPARTMENTS, 29 }, + { PHASE_SWEEP_TYPES, "Sweep type information", PHASE_SWEEP_COMPARTMENTS, 30 }, + { PHASE_SWEEP_TYPES_BEGIN, "Sweep type tables and compilations", PHASE_SWEEP_TYPES, 31 }, + { PHASE_SWEEP_TYPES_END, "Free type arena", PHASE_SWEEP_TYPES, 32 }, + { PHASE_SWEEP_OBJECT, "Sweep Object", PHASE_SWEEP, 33 }, + { PHASE_SWEEP_STRING, "Sweep String", PHASE_SWEEP, 34 }, + { PHASE_SWEEP_SCRIPT, "Sweep Script", PHASE_SWEEP, 35 }, + { PHASE_SWEEP_SCOPE, "Sweep Scope", PHASE_SWEEP, 59 }, + { PHASE_SWEEP_SHAPE, "Sweep Shape", PHASE_SWEEP, 36 }, + { PHASE_SWEEP_JITCODE, "Sweep JIT code", PHASE_SWEEP, 37 }, + { PHASE_FINALIZE_END, "Finalize End Callback", PHASE_SWEEP, 38 }, + { PHASE_DESTROY, "Deallocate", PHASE_SWEEP, 39 }, + { PHASE_COMPACT, "Compact", PHASE_NO_PARENT, 40 }, + { PHASE_COMPACT_MOVE, "Compact Move", PHASE_COMPACT, 41 }, + { PHASE_COMPACT_UPDATE, "Compact Update", PHASE_COMPACT, 42 }, + /* PHASE_MARK_ROOTS */ + { PHASE_COMPACT_UPDATE_CELLS, "Compact Update Cells", PHASE_COMPACT_UPDATE, 43 }, + { PHASE_GC_END, "End Callback", PHASE_NO_PARENT, 44 }, + { PHASE_MINOR_GC, "All Minor GCs", PHASE_NO_PARENT, 45 }, + /* PHASE_MARK_ROOTS */ + { PHASE_EVICT_NURSERY, "Minor GCs to Evict Nursery", PHASE_NO_PARENT, 46 }, + /* PHASE_MARK_ROOTS */ + { PHASE_TRACE_HEAP, "Trace Heap", PHASE_NO_PARENT, 47 }, + /* PHASE_MARK_ROOTS */ + { PHASE_BARRIER, "Barriers", PHASE_NO_PARENT, 55 }, + { PHASE_UNMARK_GRAY, "Unmark gray", PHASE_BARRIER, 56 }, + { PHASE_MARK_ROOTS, "Mark Roots", PHASE_MULTI_PARENTS, 48 }, + { PHASE_BUFFER_GRAY_ROOTS, "Buffer Gray Roots", PHASE_MARK_ROOTS, 49 }, + { PHASE_MARK_CCWS, "Mark Cross Compartment Wrappers", PHASE_MARK_ROOTS, 50 }, + { PHASE_MARK_STACK, "Mark C and JS stacks", PHASE_MARK_ROOTS, 51 }, + { PHASE_MARK_RUNTIME_DATA, "Mark Runtime-wide Data", PHASE_MARK_ROOTS, 52 }, + { PHASE_MARK_EMBEDDING, "Mark Embedding", PHASE_MARK_ROOTS, 53 }, + { PHASE_MARK_COMPARTMENTS, "Mark Compartments", PHASE_MARK_ROOTS, 54 }, + { PHASE_PURGE_SHAPE_TABLES, "Purge ShapeTables", PHASE_NO_PARENT, 60 }, + + { PHASE_LIMIT, nullptr, PHASE_NO_PARENT, 60 } + + // Current number of telemetryBuckets is 60. If you insert new phases + // somewhere, start at that number and count up. Do not change any existing + // numbers. +}; + +static ExtraPhaseInfo phaseExtra[PHASE_LIMIT] = { { 0, 0 } }; + +// Mapping from all nodes with a multi-parented child to a Vector of all +// multi-parented children and their descendants. (Single-parented children will +// not show up in this list.) +static mozilla::Vector<Phase, 0, SystemAllocPolicy> dagDescendants[Statistics::NumTimingArrays]; + +// Preorder iterator over all phases in the expanded tree. Positions are +// returned as <phase,dagSlot> pairs (dagSlot will be zero aka PHASE_DAG_NONE +// for the top nodes with a single path from the parent, and 1 or more for +// nodes in multiparented subtrees). +struct AllPhaseIterator { + // If 'descendants' is empty, the current Phase position. + int current; + + // The depth of the current multiparented node that we are processing, or + // zero if we are pointing to the top portion of the tree. + int baseLevel; + + // When looking at multiparented descendants, the dag slot (index into + // PhaseTimeTables) containing the entries for the current parent. + size_t activeSlot; + + // When iterating over a multiparented subtree, the list of (remaining) + // subtree nodes. + mozilla::Vector<Phase, 0, SystemAllocPolicy>::Range descendants; + + explicit AllPhaseIterator(const Statistics::PhaseTimeTable table) + : current(0) + , baseLevel(0) + , activeSlot(PHASE_DAG_NONE) + , descendants(dagDescendants[PHASE_DAG_NONE].all()) /* empty range */ + { + } + + void get(Phase* phase, size_t* dagSlot, size_t* level = nullptr) { + MOZ_ASSERT(!done()); + *dagSlot = activeSlot; + *phase = descendants.empty() ? Phase(current) : descendants.front(); + if (level) + *level = phaseExtra[*phase].depth + baseLevel; + } + + void advance() { + MOZ_ASSERT(!done()); + + if (!descendants.empty()) { + // Currently iterating over a multiparented subtree. + descendants.popFront(); + if (!descendants.empty()) + return; + + // Just before leaving the last child, reset the iterator to look + // at "main" phases (in PHASE_DAG_NONE) instead of multiparented + // subtree phases. + ++current; + activeSlot = PHASE_DAG_NONE; + baseLevel = 0; + return; + } + + if (phaseExtra[current].dagSlot != PHASE_DAG_NONE) { + // The current phase has a shared subtree. Load them up into + // 'descendants' and advance to the first child. + activeSlot = phaseExtra[current].dagSlot; + descendants = dagDescendants[activeSlot].all(); + MOZ_ASSERT(!descendants.empty()); + baseLevel += phaseExtra[current].depth + 1; + return; + } + + ++current; + } + + bool done() const { + return phases[current].parent == PHASE_MULTI_PARENTS; + } +}; + +void +Statistics::gcDuration(int64_t* total, int64_t* maxPause) const +{ + *total = *maxPause = 0; + for (const SliceData* slice = slices.begin(); slice != slices.end(); slice++) { + *total += slice->duration(); + if (slice->duration() > *maxPause) + *maxPause = slice->duration(); + } + if (*maxPause > maxPauseInInterval) + maxPauseInInterval = *maxPause; +} + +void +Statistics::sccDurations(int64_t* total, int64_t* maxPause) +{ + *total = *maxPause = 0; + for (size_t i = 0; i < sccTimes.length(); i++) { + *total += sccTimes[i]; + *maxPause = Max(*maxPause, sccTimes[i]); + } +} + +typedef Vector<UniqueChars, 8, SystemAllocPolicy> FragmentVector; + +static UniqueChars +Join(const FragmentVector& fragments, const char* separator = "") { + const size_t separatorLength = strlen(separator); + size_t length = 0; + for (size_t i = 0; i < fragments.length(); ++i) { + length += fragments[i] ? strlen(fragments[i].get()) : 0; + if (i < (fragments.length() - 1)) + length += separatorLength; + } + + char* joined = js_pod_malloc<char>(length + 1); + joined[length] = '\0'; + + char* cursor = joined; + for (size_t i = 0; i < fragments.length(); ++i) { + if (fragments[i]) + strcpy(cursor, fragments[i].get()); + cursor += fragments[i] ? strlen(fragments[i].get()) : 0; + if (i < (fragments.length() - 1)) { + if (separatorLength) + strcpy(cursor, separator); + cursor += separatorLength; + } + } + + return UniqueChars(joined); +} + +static int64_t +SumChildTimes(size_t phaseSlot, Phase phase, const Statistics::PhaseTimeTable phaseTimes) +{ + // Sum the contributions from single-parented children. + int64_t total = 0; + size_t depth = phaseExtra[phase].depth; + for (unsigned i = phase + 1; i < PHASE_LIMIT && phaseExtra[i].depth > depth; i++) { + if (phases[i].parent == phase) + total += phaseTimes[phaseSlot][i]; + } + + // Sum the contributions from multi-parented children. + size_t dagSlot = phaseExtra[phase].dagSlot; + if (dagSlot != PHASE_DAG_NONE) { + for (auto edge : dagChildEdges) { + if (edge.parent == phase) + total += phaseTimes[dagSlot][edge.child]; + } + } + return total; +} + +UniqueChars +Statistics::formatCompactSliceMessage() const +{ + // Skip if we OOM'ed. + if (slices.length() == 0) + return UniqueChars(nullptr); + + const size_t index = slices.length() - 1; + const SliceData& slice = slices[index]; + + char budgetDescription[200]; + slice.budget.describe(budgetDescription, sizeof(budgetDescription) - 1); + + const char* format = + "GC Slice %u - Pause: %.3fms of %s budget (@ %.3fms); Reason: %s; Reset: %s%s; Times: "; + char buffer[1024]; + SprintfLiteral(buffer, format, index, + t(slice.duration()), budgetDescription, t(slice.start - slices[0].start), + ExplainReason(slice.reason), + slice.wasReset() ? "yes - " : "no", + slice.wasReset() ? ExplainAbortReason(slice.resetReason) : ""); + + FragmentVector fragments; + if (!fragments.append(DuplicateString(buffer)) || + !fragments.append(formatCompactSlicePhaseTimes(slices[index].phaseTimes))) + { + return UniqueChars(nullptr); + } + return Join(fragments); +} + +UniqueChars +Statistics::formatCompactSummaryMessage() const +{ + const double bytesPerMiB = 1024 * 1024; + + FragmentVector fragments; + if (!fragments.append(DuplicateString("Summary - "))) + return UniqueChars(nullptr); + + int64_t total, longest; + gcDuration(&total, &longest); + + const double mmu20 = computeMMU(20 * PRMJ_USEC_PER_MSEC); + const double mmu50 = computeMMU(50 * PRMJ_USEC_PER_MSEC); + + char buffer[1024]; + if (!nonincremental()) { + SprintfLiteral(buffer, + "Max Pause: %.3fms; MMU 20ms: %.1f%%; MMU 50ms: %.1f%%; Total: %.3fms; ", + t(longest), mmu20 * 100., mmu50 * 100., t(total)); + } else { + SprintfLiteral(buffer, "Non-Incremental: %.3fms (%s); ", + t(total), ExplainAbortReason(nonincrementalReason_)); + } + if (!fragments.append(DuplicateString(buffer))) + return UniqueChars(nullptr); + + SprintfLiteral(buffer, + "Zones: %d of %d (-%d); Compartments: %d of %d (-%d); HeapSize: %.3f MiB; " \ + "HeapChange (abs): %+d (%d); ", + zoneStats.collectedZoneCount, zoneStats.zoneCount, zoneStats.sweptZoneCount, + zoneStats.collectedCompartmentCount, zoneStats.compartmentCount, + zoneStats.sweptCompartmentCount, + double(preBytes) / bytesPerMiB, + counts[STAT_NEW_CHUNK] - counts[STAT_DESTROY_CHUNK], + counts[STAT_NEW_CHUNK] + counts[STAT_DESTROY_CHUNK]); + if (!fragments.append(DuplicateString(buffer))) + return UniqueChars(nullptr); + + MOZ_ASSERT_IF(counts[STAT_ARENA_RELOCATED], gckind == GC_SHRINK); + if (gckind == GC_SHRINK) { + SprintfLiteral(buffer, + "Kind: %s; Relocated: %.3f MiB; ", + ExplainInvocationKind(gckind), + double(ArenaSize * counts[STAT_ARENA_RELOCATED]) / bytesPerMiB); + if (!fragments.append(DuplicateString(buffer))) + return UniqueChars(nullptr); + } + + return Join(fragments); +} + +UniqueChars +Statistics::formatCompactSlicePhaseTimes(const PhaseTimeTable phaseTimes) const +{ + static const int64_t MaxUnaccountedTimeUS = 100; + + FragmentVector fragments; + char buffer[128]; + for (AllPhaseIterator iter(phaseTimes); !iter.done(); iter.advance()) { + Phase phase; + size_t dagSlot; + size_t level; + iter.get(&phase, &dagSlot, &level); + MOZ_ASSERT(level < 4); + + int64_t ownTime = phaseTimes[dagSlot][phase]; + int64_t childTime = SumChildTimes(dagSlot, phase, phaseTimes); + if (ownTime > MaxUnaccountedTimeUS) { + SprintfLiteral(buffer, "%s: %.3fms", phases[phase].name, t(ownTime)); + if (!fragments.append(DuplicateString(buffer))) + return UniqueChars(nullptr); + + if (childTime && (ownTime - childTime) > MaxUnaccountedTimeUS) { + MOZ_ASSERT(level < 3); + SprintfLiteral(buffer, "%s: %.3fms", "Other", t(ownTime - childTime)); + if (!fragments.append(DuplicateString(buffer))) + return UniqueChars(nullptr); + } + } + } + return Join(fragments, ", "); +} + +UniqueChars +Statistics::formatDetailedMessage() +{ + FragmentVector fragments; + + if (!fragments.append(formatDetailedDescription())) + return UniqueChars(nullptr); + + if (slices.length() > 1) { + for (unsigned i = 0; i < slices.length(); i++) { + if (!fragments.append(formatDetailedSliceDescription(i, slices[i]))) + return UniqueChars(nullptr); + if (!fragments.append(formatDetailedPhaseTimes(slices[i].phaseTimes))) + return UniqueChars(nullptr); + } + } + if (!fragments.append(formatDetailedTotals())) + return UniqueChars(nullptr); + if (!fragments.append(formatDetailedPhaseTimes(phaseTimes))) + return UniqueChars(nullptr); + + return Join(fragments); +} + +UniqueChars +Statistics::formatDetailedDescription() +{ + const double bytesPerMiB = 1024 * 1024; + + int64_t sccTotal, sccLongest; + sccDurations(&sccTotal, &sccLongest); + + double mmu20 = computeMMU(20 * PRMJ_USEC_PER_MSEC); + double mmu50 = computeMMU(50 * PRMJ_USEC_PER_MSEC); + + const char* format = +"=================================================================\n\ + Invocation Kind: %s\n\ + Reason: %s\n\ + Incremental: %s%s\n\ + Zones Collected: %d of %d (-%d)\n\ + Compartments Collected: %d of %d (-%d)\n\ + MinorGCs since last GC: %d\n\ + Store Buffer Overflows: %d\n\ + MMU 20ms:%.1f%%; 50ms:%.1f%%\n\ + SCC Sweep Total (MaxPause): %.3fms (%.3fms)\n\ + HeapSize: %.3f MiB\n\ + Chunk Delta (magnitude): %+d (%d)\n\ + Arenas Relocated: %.3f MiB\n\ +"; + char buffer[1024]; + SprintfLiteral(buffer, format, + ExplainInvocationKind(gckind), + ExplainReason(slices[0].reason), + nonincremental() ? "no - " : "yes", + nonincremental() ? ExplainAbortReason(nonincrementalReason_) : "", + zoneStats.collectedZoneCount, zoneStats.zoneCount, zoneStats.sweptZoneCount, + zoneStats.collectedCompartmentCount, zoneStats.compartmentCount, + zoneStats.sweptCompartmentCount, + counts[STAT_MINOR_GC], + counts[STAT_STOREBUFFER_OVERFLOW], + mmu20 * 100., mmu50 * 100., + t(sccTotal), t(sccLongest), + double(preBytes) / bytesPerMiB, + counts[STAT_NEW_CHUNK] - counts[STAT_DESTROY_CHUNK], + counts[STAT_NEW_CHUNK] + counts[STAT_DESTROY_CHUNK], + double(ArenaSize * counts[STAT_ARENA_RELOCATED]) / bytesPerMiB); + return DuplicateString(buffer); +} + +UniqueChars +Statistics::formatDetailedSliceDescription(unsigned i, const SliceData& slice) +{ + char budgetDescription[200]; + slice.budget.describe(budgetDescription, sizeof(budgetDescription) - 1); + + const char* format = +"\ + ---- Slice %u ----\n\ + Reason: %s\n\ + Reset: %s%s\n\ + State: %s -> %s\n\ + Page Faults: %ld\n\ + Pause: %.3fms of %s budget (@ %.3fms)\n\ +"; + char buffer[1024]; + SprintfLiteral(buffer, format, i, ExplainReason(slice.reason), + slice.wasReset() ? "yes - " : "no", + slice.wasReset() ? ExplainAbortReason(slice.resetReason) : "", + gc::StateName(slice.initialState), gc::StateName(slice.finalState), + uint64_t(slice.endFaults - slice.startFaults), + t(slice.duration()), budgetDescription, t(slice.start - slices[0].start)); + return DuplicateString(buffer); +} + +UniqueChars +Statistics::formatDetailedPhaseTimes(const PhaseTimeTable phaseTimes) +{ + static const char* LevelToIndent[] = { "", " ", " ", " " }; + static const int64_t MaxUnaccountedChildTimeUS = 50; + + FragmentVector fragments; + char buffer[128]; + for (AllPhaseIterator iter(phaseTimes); !iter.done(); iter.advance()) { + Phase phase; + size_t dagSlot; + size_t level; + iter.get(&phase, &dagSlot, &level); + MOZ_ASSERT(level < 4); + + int64_t ownTime = phaseTimes[dagSlot][phase]; + int64_t childTime = SumChildTimes(dagSlot, phase, phaseTimes); + if (ownTime > 0) { + SprintfLiteral(buffer, " %s%s: %.3fms\n", + LevelToIndent[level], phases[phase].name, t(ownTime)); + if (!fragments.append(DuplicateString(buffer))) + return UniqueChars(nullptr); + + if (childTime && (ownTime - childTime) > MaxUnaccountedChildTimeUS) { + MOZ_ASSERT(level < 3); + SprintfLiteral(buffer, " %s%s: %.3fms\n", + LevelToIndent[level + 1], "Other", t(ownTime - childTime)); + if (!fragments.append(DuplicateString(buffer))) + return UniqueChars(nullptr); + } + } + } + return Join(fragments); +} + +UniqueChars +Statistics::formatDetailedTotals() +{ + int64_t total, longest; + gcDuration(&total, &longest); + + const char* format = +"\ + ---- Totals ----\n\ + Total Time: %.3fms\n\ + Max Pause: %.3fms\n\ +"; + char buffer[1024]; + SprintfLiteral(buffer, format, t(total), t(longest)); + return DuplicateString(buffer); +} + +UniqueChars +Statistics::formatJsonMessage(uint64_t timestamp) +{ + MOZ_ASSERT(!aborted); + + FragmentVector fragments; + + if (!fragments.append(DuplicateString("{")) || + !fragments.append(formatJsonDescription(timestamp)) || + !fragments.append(DuplicateString("\"slices\":["))) + { + return UniqueChars(nullptr); + } + + for (unsigned i = 0; i < slices.length(); i++) { + if (!fragments.append(DuplicateString("{")) || + !fragments.append(formatJsonSliceDescription(i, slices[i])) || + !fragments.append(DuplicateString("\"times\":{")) || + !fragments.append(formatJsonPhaseTimes(slices[i].phaseTimes)) || + !fragments.append(DuplicateString("}}")) || + (i < (slices.length() - 1) && !fragments.append(DuplicateString(",")))) + { + return UniqueChars(nullptr); + } + } + + if (!fragments.append(DuplicateString("],\"totals\":{")) || + !fragments.append(formatJsonPhaseTimes(phaseTimes)) || + !fragments.append(DuplicateString("}}"))) + { + return UniqueChars(nullptr); + } + + return Join(fragments); +} + +UniqueChars +Statistics::formatJsonDescription(uint64_t timestamp) +{ + int64_t total, longest; + gcDuration(&total, &longest); + + int64_t sccTotal, sccLongest; + sccDurations(&sccTotal, &sccLongest); + + double mmu20 = computeMMU(20 * PRMJ_USEC_PER_MSEC); + double mmu50 = computeMMU(50 * PRMJ_USEC_PER_MSEC); + + const char *format = + "\"timestamp\":%llu," + "\"max_pause\":%llu.%03llu," + "\"total_time\":%llu.%03llu," + "\"zones_collected\":%d," + "\"total_zones\":%d," + "\"total_compartments\":%d," + "\"minor_gcs\":%d," + "\"store_buffer_overflows\":%d," + "\"mmu_20ms\":%d," + "\"mmu_50ms\":%d," + "\"scc_sweep_total\":%llu.%03llu," + "\"scc_sweep_max_pause\":%llu.%03llu," + "\"nonincremental_reason\":\"%s\"," + "\"allocated\":%u," + "\"added_chunks\":%d," + "\"removed_chunks\":%d,"; + char buffer[1024]; + SprintfLiteral(buffer, format, + (unsigned long long)timestamp, + longest / 1000, longest % 1000, + total / 1000, total % 1000, + zoneStats.collectedZoneCount, + zoneStats.zoneCount, + zoneStats.compartmentCount, + counts[STAT_MINOR_GC], + counts[STAT_STOREBUFFER_OVERFLOW], + int(mmu20 * 100), + int(mmu50 * 100), + sccTotal / 1000, sccTotal % 1000, + sccLongest / 1000, sccLongest % 1000, + ExplainAbortReason(nonincrementalReason_), + unsigned(preBytes / 1024 / 1024), + counts[STAT_NEW_CHUNK], + counts[STAT_DESTROY_CHUNK]); + return DuplicateString(buffer); +} + +UniqueChars +Statistics::formatJsonSliceDescription(unsigned i, const SliceData& slice) +{ + int64_t duration = slice.duration(); + int64_t when = slice.start - slices[0].start; + char budgetDescription[200]; + slice.budget.describe(budgetDescription, sizeof(budgetDescription) - 1); + int64_t pageFaults = slice.endFaults - slice.startFaults; + + const char* format = + "\"slice\":%d," + "\"pause\":%llu.%03llu," + "\"when\":%llu.%03llu," + "\"reason\":\"%s\"," + "\"initial_state\":\"%s\"," + "\"final_state\":\"%s\"," + "\"budget\":\"%s\"," + "\"page_faults\":%llu," + "\"start_timestamp\":%llu," + "\"end_timestamp\":%llu,"; + char buffer[1024]; + SprintfLiteral(buffer, format, + i, + duration / 1000, duration % 1000, + when / 1000, when % 1000, + ExplainReason(slice.reason), + gc::StateName(slice.initialState), + gc::StateName(slice.finalState), + budgetDescription, + pageFaults, + slice.start, + slice.end); + return DuplicateString(buffer); +} + +UniqueChars +FilterJsonKey(const char*const buffer) +{ + char* mut = strdup(buffer); + char* c = mut; + while (*c) { + if (!isalpha(*c)) + *c = '_'; + else if (isupper(*c)) + *c = tolower(*c); + ++c; + } + return UniqueChars(mut); +} + +UniqueChars +Statistics::formatJsonPhaseTimes(const PhaseTimeTable phaseTimes) +{ + FragmentVector fragments; + char buffer[128]; + for (AllPhaseIterator iter(phaseTimes); !iter.done(); iter.advance()) { + Phase phase; + size_t dagSlot; + iter.get(&phase, &dagSlot); + + UniqueChars name = FilterJsonKey(phases[phase].name); + int64_t ownTime = phaseTimes[dagSlot][phase]; + if (ownTime > 0) { + SprintfLiteral(buffer, "\"%s\":%" PRId64 ".%03" PRId64, + name.get(), ownTime / 1000, ownTime % 1000); + + if (!fragments.append(DuplicateString(buffer))) + return UniqueChars(nullptr); + } + } + return Join(fragments, ","); +} + +Statistics::Statistics(JSRuntime* rt) + : runtime(rt), + startupTime(PRMJ_Now()), + fp(nullptr), + gcDepth(0), + nonincrementalReason_(gc::AbortReason::None), + timedGCStart(0), + preBytes(0), + maxPauseInInterval(0), + phaseNestingDepth(0), + activeDagSlot(PHASE_DAG_NONE), + suspended(0), + sliceCallback(nullptr), + nurseryCollectionCallback(nullptr), + aborted(false), + enableProfiling_(false), + sliceCount_(0) +{ + PodArrayZero(phaseTotals); + PodArrayZero(counts); + PodArrayZero(phaseStartTimes); + for (auto d : MakeRange(NumTimingArrays)) + PodArrayZero(phaseTimes[d]); + + const char* env = getenv("MOZ_GCTIMER"); + if (env) { + if (strcmp(env, "none") == 0) { + fp = nullptr; + } else if (strcmp(env, "stdout") == 0) { + fp = stdout; + } else if (strcmp(env, "stderr") == 0) { + fp = stderr; + } else { + fp = fopen(env, "a"); + if (!fp) + MOZ_CRASH("Failed to open MOZ_GCTIMER log file."); + } + } + + env = getenv("JS_GC_PROFILE"); + if (env) { + if (0 == strcmp(env, "help")) { + fprintf(stderr, "JS_GC_PROFILE=N\n" + "\tReport major GC's taking more than N milliseconds.\n"); + exit(0); + } + enableProfiling_ = true; + profileThreshold_ = atoi(env); + } + + PodZero(&totalTimes_); +} + +Statistics::~Statistics() +{ + if (fp && fp != stdout && fp != stderr) + fclose(fp); +} + +/* static */ bool +Statistics::initialize() +{ + for (size_t i = 0; i < PHASE_LIMIT; i++) { + MOZ_ASSERT(phases[i].index == i); + for (size_t j = 0; j < PHASE_LIMIT; j++) + MOZ_ASSERT_IF(i != j, phases[i].telemetryBucket != phases[j].telemetryBucket); + } + + // Create a static table of descendants for every phase with multiple + // children. This assumes that all descendants come linearly in the + // list, which is reasonable since full dags are not supported; any + // path from the leaf to the root must encounter at most one node with + // multiple parents. + size_t dagSlot = 0; + for (size_t i = 0; i < mozilla::ArrayLength(dagChildEdges); i++) { + Phase parent = dagChildEdges[i].parent; + if (!phaseExtra[parent].dagSlot) + phaseExtra[parent].dagSlot = ++dagSlot; + + Phase child = dagChildEdges[i].child; + MOZ_ASSERT(phases[child].parent == PHASE_MULTI_PARENTS); + int j = child; + do { + if (!dagDescendants[phaseExtra[parent].dagSlot].append(Phase(j))) + return false; + j++; + } while (j != PHASE_LIMIT && phases[j].parent != PHASE_MULTI_PARENTS); + } + MOZ_ASSERT(dagSlot <= MaxMultiparentPhases - 1); + + // Fill in the depth of each node in the tree. Multi-parented nodes + // have depth 0. + mozilla::Vector<Phase, 0, SystemAllocPolicy> stack; + if (!stack.append(PHASE_LIMIT)) // Dummy entry to avoid special-casing the first node + return false; + for (int i = 0; i < PHASE_LIMIT; i++) { + if (phases[i].parent == PHASE_NO_PARENT || + phases[i].parent == PHASE_MULTI_PARENTS) + { + stack.clear(); + } else { + while (stack.back() != phases[i].parent) + stack.popBack(); + } + phaseExtra[i].depth = stack.length(); + if (!stack.append(Phase(i))) + return false; + } + + return true; +} + +JS::GCSliceCallback +Statistics::setSliceCallback(JS::GCSliceCallback newCallback) +{ + JS::GCSliceCallback oldCallback = sliceCallback; + sliceCallback = newCallback; + return oldCallback; +} + +JS::GCNurseryCollectionCallback +Statistics::setNurseryCollectionCallback(JS::GCNurseryCollectionCallback newCallback) +{ + auto oldCallback = nurseryCollectionCallback; + nurseryCollectionCallback = newCallback; + return oldCallback; +} + +int64_t +Statistics::clearMaxGCPauseAccumulator() +{ + int64_t prior = maxPauseInInterval; + maxPauseInInterval = 0; + return prior; +} + +int64_t +Statistics::getMaxGCPauseSinceClear() +{ + return maxPauseInInterval; +} + +// Sum up the time for a phase, including instances of the phase with different +// parents. +static int64_t +SumPhase(Phase phase, const Statistics::PhaseTimeTable times) +{ + int64_t sum = 0; + for (auto i : MakeRange(Statistics::NumTimingArrays)) + sum += times[i][phase]; + return sum; +} + +static Phase +LongestPhase(const Statistics::PhaseTimeTable times) +{ + int64_t longestTime = 0; + Phase longestPhase = PHASE_NONE; + for (size_t i = 0; i < PHASE_LIMIT; ++i) { + int64_t phaseTime = SumPhase(Phase(i), times); + if (phaseTime > longestTime) { + longestTime = phaseTime; + longestPhase = Phase(i); + } + } + return longestPhase; +} + +void +Statistics::printStats() +{ + if (aborted) { + fprintf(fp, "OOM during GC statistics collection. The report is unavailable for this GC.\n"); + } else { + UniqueChars msg = formatDetailedMessage(); + if (msg) + fprintf(fp, "GC(T+%.3fs) %s\n", t(slices[0].start - startupTime) / 1000.0, msg.get()); + } + fflush(fp); +} + +void +Statistics::beginGC(JSGCInvocationKind kind) +{ + slices.clearAndFree(); + sccTimes.clearAndFree(); + gckind = kind; + nonincrementalReason_ = gc::AbortReason::None; + + preBytes = runtime->gc.usage.gcBytes(); +} + +void +Statistics::endGC() +{ + for (auto j : MakeRange(NumTimingArrays)) + for (int i = 0; i < PHASE_LIMIT; i++) + phaseTotals[j][i] += phaseTimes[j][i]; + + int64_t total, longest; + gcDuration(&total, &longest); + + int64_t sccTotal, sccLongest; + sccDurations(&sccTotal, &sccLongest); + + runtime->addTelemetry(JS_TELEMETRY_GC_IS_ZONE_GC, !zoneStats.isCollectingAllZones()); + runtime->addTelemetry(JS_TELEMETRY_GC_MS, t(total)); + runtime->addTelemetry(JS_TELEMETRY_GC_MAX_PAUSE_MS, t(longest)); + int64_t markTotal = SumPhase(PHASE_MARK, phaseTimes); + int64_t markRootsTotal = SumPhase(PHASE_MARK_ROOTS, phaseTimes); + runtime->addTelemetry(JS_TELEMETRY_GC_MARK_MS, t(markTotal)); + runtime->addTelemetry(JS_TELEMETRY_GC_SWEEP_MS, t(phaseTimes[PHASE_DAG_NONE][PHASE_SWEEP])); + if (runtime->gc.isCompactingGc()) { + runtime->addTelemetry(JS_TELEMETRY_GC_COMPACT_MS, + t(phaseTimes[PHASE_DAG_NONE][PHASE_COMPACT])); + } + runtime->addTelemetry(JS_TELEMETRY_GC_MARK_ROOTS_MS, t(markRootsTotal)); + runtime->addTelemetry(JS_TELEMETRY_GC_MARK_GRAY_MS, t(phaseTimes[PHASE_DAG_NONE][PHASE_SWEEP_MARK_GRAY])); + runtime->addTelemetry(JS_TELEMETRY_GC_NON_INCREMENTAL, nonincremental()); + if (nonincremental()) + runtime->addTelemetry(JS_TELEMETRY_GC_NON_INCREMENTAL_REASON, uint32_t(nonincrementalReason_)); + runtime->addTelemetry(JS_TELEMETRY_GC_INCREMENTAL_DISABLED, !runtime->gc.isIncrementalGCAllowed()); + runtime->addTelemetry(JS_TELEMETRY_GC_SCC_SWEEP_TOTAL_MS, t(sccTotal)); + runtime->addTelemetry(JS_TELEMETRY_GC_SCC_SWEEP_MAX_PAUSE_MS, t(sccLongest)); + + if (!aborted) { + double mmu50 = computeMMU(50 * PRMJ_USEC_PER_MSEC); + runtime->addTelemetry(JS_TELEMETRY_GC_MMU_50, mmu50 * 100); + } + + if (fp) + printStats(); + + // Clear the OOM flag but only if we are not in a nested GC. + if (gcDepth == 1) + aborted = false; +} + +void +Statistics::beginNurseryCollection(JS::gcreason::Reason reason) +{ + count(STAT_MINOR_GC); + if (nurseryCollectionCallback) { + (*nurseryCollectionCallback)(runtime->contextFromMainThread(), + JS::GCNurseryProgress::GC_NURSERY_COLLECTION_START, + reason); + } +} + +void +Statistics::endNurseryCollection(JS::gcreason::Reason reason) +{ + if (nurseryCollectionCallback) { + (*nurseryCollectionCallback)(runtime->contextFromMainThread(), + JS::GCNurseryProgress::GC_NURSERY_COLLECTION_END, + reason); + } +} + +void +Statistics::beginSlice(const ZoneGCStats& zoneStats, JSGCInvocationKind gckind, + SliceBudget budget, JS::gcreason::Reason reason) +{ + gcDepth++; + this->zoneStats = zoneStats; + + bool first = !runtime->gc.isIncrementalGCInProgress(); + if (first) + beginGC(gckind); + + SliceData data(budget, reason, PRMJ_Now(), JS_GetCurrentEmbedderTime(), GetPageFaultCount(), + runtime->gc.state()); + if (!slices.append(data)) { + // If we are OOM, set a flag to indicate we have missing slice data. + aborted = true; + return; + } + + runtime->addTelemetry(JS_TELEMETRY_GC_REASON, reason); + + // Slice callbacks should only fire for the outermost level. + if (gcDepth == 1) { + bool wasFullGC = zoneStats.isCollectingAllZones(); + if (sliceCallback) + (*sliceCallback)(runtime->contextFromMainThread(), + first ? JS::GC_CYCLE_BEGIN : JS::GC_SLICE_BEGIN, + JS::GCDescription(!wasFullGC, gckind, reason)); + } +} + +void +Statistics::endSlice() +{ + if (!aborted) { + slices.back().end = PRMJ_Now(); + slices.back().endTimestamp = JS_GetCurrentEmbedderTime(); + slices.back().endFaults = GetPageFaultCount(); + slices.back().finalState = runtime->gc.state(); + + int64_t sliceTime = slices.back().end - slices.back().start; + runtime->addTelemetry(JS_TELEMETRY_GC_SLICE_MS, t(sliceTime)); + runtime->addTelemetry(JS_TELEMETRY_GC_RESET, slices.back().wasReset()); + if (slices.back().wasReset()) + runtime->addTelemetry(JS_TELEMETRY_GC_RESET_REASON, uint32_t(slices.back().resetReason)); + + if (slices.back().budget.isTimeBudget()) { + int64_t budget_ms = slices.back().budget.timeBudget.budget; + runtime->addTelemetry(JS_TELEMETRY_GC_BUDGET_MS, budget_ms); + if (budget_ms == runtime->gc.defaultSliceBudget()) + runtime->addTelemetry(JS_TELEMETRY_GC_ANIMATION_MS, t(sliceTime)); + + // Record any phase that goes more than 2x over its budget. + if (sliceTime > 2 * budget_ms * 1000) { + Phase longest = LongestPhase(slices.back().phaseTimes); + runtime->addTelemetry(JS_TELEMETRY_GC_SLOW_PHASE, phases[longest].telemetryBucket); + } + } + + sliceCount_++; + } + + bool last = !runtime->gc.isIncrementalGCInProgress(); + if (last) + endGC(); + + if (enableProfiling_ && !aborted && slices.back().duration() >= profileThreshold_) + printSliceProfile(); + + // Slice callbacks should only fire for the outermost level. + if (gcDepth == 1 && !aborted) { + bool wasFullGC = zoneStats.isCollectingAllZones(); + if (sliceCallback) + (*sliceCallback)(runtime->contextFromMainThread(), + last ? JS::GC_CYCLE_END : JS::GC_SLICE_END, + JS::GCDescription(!wasFullGC, gckind, slices.back().reason)); + } + + /* Do this after the slice callback since it uses these values. */ + if (last) { + PodArrayZero(counts); + + // Clear the timers at the end of a GC because we accumulate time in + // between GCs for some (which come before PHASE_GC_BEGIN in the list.) + PodZero(&phaseStartTimes[PHASE_GC_BEGIN], PHASE_LIMIT - PHASE_GC_BEGIN); + for (size_t d = PHASE_DAG_NONE; d < NumTimingArrays; d++) + PodZero(&phaseTimes[d][PHASE_GC_BEGIN], PHASE_LIMIT - PHASE_GC_BEGIN); + } + + gcDepth--; + MOZ_ASSERT(gcDepth >= 0); +} + +bool +Statistics::startTimingMutator() +{ + if (phaseNestingDepth != 0) { + // Should only be called from outside of GC. + MOZ_ASSERT(phaseNestingDepth == 1); + MOZ_ASSERT(phaseNesting[0] == PHASE_MUTATOR); + return false; + } + + MOZ_ASSERT(suspended == 0); + + timedGCTime = 0; + phaseStartTimes[PHASE_MUTATOR] = 0; + phaseTimes[PHASE_DAG_NONE][PHASE_MUTATOR] = 0; + timedGCStart = 0; + + beginPhase(PHASE_MUTATOR); + return true; +} + +bool +Statistics::stopTimingMutator(double& mutator_ms, double& gc_ms) +{ + // This should only be called from outside of GC, while timing the mutator. + if (phaseNestingDepth != 1 || phaseNesting[0] != PHASE_MUTATOR) + return false; + + endPhase(PHASE_MUTATOR); + mutator_ms = t(phaseTimes[PHASE_DAG_NONE][PHASE_MUTATOR]); + gc_ms = t(timedGCTime); + + return true; +} + +void +Statistics::suspendPhases(Phase suspension) +{ + MOZ_ASSERT(suspension == PHASE_EXPLICIT_SUSPENSION || suspension == PHASE_IMPLICIT_SUSPENSION); + while (phaseNestingDepth) { + MOZ_ASSERT(suspended < mozilla::ArrayLength(suspendedPhases)); + Phase parent = phaseNesting[phaseNestingDepth - 1]; + suspendedPhases[suspended++] = parent; + recordPhaseEnd(parent); + } + suspendedPhases[suspended++] = suspension; +} + +void +Statistics::resumePhases() +{ + DebugOnly<Phase> popped = suspendedPhases[--suspended]; + MOZ_ASSERT(popped == PHASE_EXPLICIT_SUSPENSION || popped == PHASE_IMPLICIT_SUSPENSION); + while (suspended && + suspendedPhases[suspended - 1] != PHASE_EXPLICIT_SUSPENSION && + suspendedPhases[suspended - 1] != PHASE_IMPLICIT_SUSPENSION) + { + Phase resumePhase = suspendedPhases[--suspended]; + if (resumePhase == PHASE_MUTATOR) + timedGCTime += PRMJ_Now() - timedGCStart; + beginPhase(resumePhase); + } +} + +void +Statistics::beginPhase(Phase phase) +{ + Phase parent = phaseNestingDepth ? phaseNesting[phaseNestingDepth - 1] : PHASE_NO_PARENT; + + // Re-entry is allowed during callbacks, so pause callback phases while + // other phases are in progress, auto-resuming after they end. As a result, + // nested GC time will not be accounted against the callback phases. + // + // Reuse this mechanism for managing PHASE_MUTATOR. + if (parent == PHASE_GC_BEGIN || parent == PHASE_GC_END || parent == PHASE_MUTATOR) { + suspendPhases(PHASE_IMPLICIT_SUSPENSION); + parent = phaseNestingDepth ? phaseNesting[phaseNestingDepth - 1] : PHASE_NO_PARENT; + } + + // Guard against any other re-entry. + MOZ_ASSERT(!phaseStartTimes[phase]); + + MOZ_ASSERT(phases[phase].index == phase); + MOZ_ASSERT(phaseNestingDepth < MAX_NESTING); + MOZ_ASSERT(phases[phase].parent == parent || phases[phase].parent == PHASE_MULTI_PARENTS); + + phaseNesting[phaseNestingDepth] = phase; + phaseNestingDepth++; + + if (phases[phase].parent == PHASE_MULTI_PARENTS) + activeDagSlot = phaseExtra[parent].dagSlot; + + phaseStartTimes[phase] = PRMJ_Now(); +} + +void +Statistics::recordPhaseEnd(Phase phase) +{ + int64_t now = PRMJ_Now(); + + if (phase == PHASE_MUTATOR) + timedGCStart = now; + + phaseNestingDepth--; + + int64_t t = now - phaseStartTimes[phase]; + if (!slices.empty()) + slices.back().phaseTimes[activeDagSlot][phase] += t; + phaseTimes[activeDagSlot][phase] += t; + phaseStartTimes[phase] = 0; +} + +void +Statistics::endPhase(Phase phase) +{ + recordPhaseEnd(phase); + + if (phases[phase].parent == PHASE_MULTI_PARENTS) + activeDagSlot = PHASE_DAG_NONE; + + // When emptying the stack, we may need to resume a callback phase + // (PHASE_GC_BEGIN/END) or return to timing the mutator (PHASE_MUTATOR). + if (phaseNestingDepth == 0 && suspended > 0 && suspendedPhases[suspended - 1] == PHASE_IMPLICIT_SUSPENSION) + resumePhases(); +} + +void +Statistics::endParallelPhase(Phase phase, const GCParallelTask* task) +{ + phaseNestingDepth--; + + if (!slices.empty()) + slices.back().phaseTimes[PHASE_DAG_NONE][phase] += task->duration(); + phaseTimes[PHASE_DAG_NONE][phase] += task->duration(); + phaseStartTimes[phase] = 0; +} + +int64_t +Statistics::beginSCC() +{ + return PRMJ_Now(); +} + +void +Statistics::endSCC(unsigned scc, int64_t start) +{ + if (scc >= sccTimes.length() && !sccTimes.resize(scc + 1)) + return; + + sccTimes[scc] += PRMJ_Now() - start; +} + +/* + * MMU (minimum mutator utilization) is a measure of how much garbage collection + * is affecting the responsiveness of the system. MMU measurements are given + * with respect to a certain window size. If we report MMU(50ms) = 80%, then + * that means that, for any 50ms window of time, at least 80% of the window is + * devoted to the mutator. In other words, the GC is running for at most 20% of + * the window, or 10ms. The GC can run multiple slices during the 50ms window + * as long as the total time it spends is at most 10ms. + */ +double +Statistics::computeMMU(int64_t window) const +{ + MOZ_ASSERT(!slices.empty()); + + int64_t gc = slices[0].end - slices[0].start; + int64_t gcMax = gc; + + if (gc >= window) + return 0.0; + + int startIndex = 0; + for (size_t endIndex = 1; endIndex < slices.length(); endIndex++) { + gc += slices[endIndex].end - slices[endIndex].start; + + while (slices[endIndex].end - slices[startIndex].end >= window) { + gc -= slices[startIndex].end - slices[startIndex].start; + startIndex++; + } + + int64_t cur = gc; + if (slices[endIndex].end - slices[startIndex].start > window) + cur -= (slices[endIndex].end - slices[startIndex].start - window); + if (cur > gcMax) + gcMax = cur; + } + + return double(window - gcMax) / window; +} + +/* static */ void +Statistics::printProfileHeader() +{ + fprintf(stderr, " %6s", "total"); +#define PRINT_PROFILE_HEADER(name, text, phase) \ + fprintf(stderr, " %6s", text); +FOR_EACH_GC_PROFILE_TIME(PRINT_PROFILE_HEADER) +#undef PRINT_PROFILE_HEADER + fprintf(stderr, "\n"); +} + +/* static */ void +Statistics::printProfileTimes(const ProfileTimes& times) +{ + for (auto time : times) + fprintf(stderr, " %6" PRIi64, time / PRMJ_USEC_PER_MSEC); + fprintf(stderr, "\n"); +} + +void +Statistics::printSliceProfile() +{ + const SliceData& slice = slices.back(); + + static int printedHeader = 0; + if ((printedHeader++ % 200) == 0) { + fprintf(stderr, "MajorGC: Reason States "); + printProfileHeader(); + } + + fprintf(stderr, "MajorGC: %20s %1d -> %1d ", + ExplainReason(slice.reason), int(slice.initialState), int(slice.finalState)); + + ProfileTimes times; + times[ProfileKey::Total] = slice.duration(); + totalTimes_[ProfileKey::Total] += times[ProfileKey::Total]; + +#define GET_PROFILE_TIME(name, text, phase) \ + times[ProfileKey::name] = slice.phaseTimes[PHASE_DAG_NONE][phase]; \ + totalTimes_[ProfileKey::name] += times[ProfileKey::name]; +FOR_EACH_GC_PROFILE_TIME(GET_PROFILE_TIME) +#undef GET_PROFILE_TIME + + printProfileTimes(times); +} + +void +Statistics::printTotalProfileTimes() +{ + if (enableProfiling_) { + fprintf(stderr, "MajorGC TOTALS: %7" PRIu64 " slices: ", sliceCount_); + printProfileTimes(totalTimes_); + } +} + diff --git a/js/src/gc/Statistics.h b/js/src/gc/Statistics.h new file mode 100644 index 000000000..c9e5871e3 --- /dev/null +++ b/js/src/gc/Statistics.h @@ -0,0 +1,505 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_Statistics_h +#define gc_Statistics_h + +#include "mozilla/EnumeratedArray.h" +#include "mozilla/IntegerRange.h" +#include "mozilla/Maybe.h" +#include "mozilla/PodOperations.h" + +#include "jsalloc.h" +#include "jsgc.h" +#include "jspubtd.h" + +#include "js/GCAPI.h" +#include "js/Vector.h" + +using mozilla::Maybe; + +namespace js { + +class GCParallelTask; + +namespace gcstats { + +enum Phase : uint8_t { + PHASE_MUTATOR, + PHASE_GC_BEGIN, + PHASE_WAIT_BACKGROUND_THREAD, + PHASE_MARK_DISCARD_CODE, + PHASE_RELAZIFY_FUNCTIONS, + PHASE_PURGE, + PHASE_MARK, + PHASE_UNMARK, + PHASE_MARK_DELAYED, + PHASE_SWEEP, + PHASE_SWEEP_MARK, + PHASE_SWEEP_MARK_TYPES, + PHASE_SWEEP_MARK_INCOMING_BLACK, + PHASE_SWEEP_MARK_WEAK, + PHASE_SWEEP_MARK_INCOMING_GRAY, + PHASE_SWEEP_MARK_GRAY, + PHASE_SWEEP_MARK_GRAY_WEAK, + PHASE_FINALIZE_START, + PHASE_WEAK_ZONEGROUP_CALLBACK, + PHASE_WEAK_COMPARTMENT_CALLBACK, + PHASE_SWEEP_ATOMS, + PHASE_SWEEP_SYMBOL_REGISTRY, + PHASE_SWEEP_COMPARTMENTS, + PHASE_SWEEP_DISCARD_CODE, + PHASE_SWEEP_INNER_VIEWS, + PHASE_SWEEP_CC_WRAPPER, + PHASE_SWEEP_BASE_SHAPE, + PHASE_SWEEP_INITIAL_SHAPE, + PHASE_SWEEP_TYPE_OBJECT, + PHASE_SWEEP_BREAKPOINT, + PHASE_SWEEP_REGEXP, + PHASE_SWEEP_MISC, + PHASE_SWEEP_TYPES, + PHASE_SWEEP_TYPES_BEGIN, + PHASE_SWEEP_TYPES_END, + PHASE_SWEEP_OBJECT, + PHASE_SWEEP_STRING, + PHASE_SWEEP_SCRIPT, + PHASE_SWEEP_SCOPE, + PHASE_SWEEP_SHAPE, + PHASE_SWEEP_JITCODE, + PHASE_FINALIZE_END, + PHASE_DESTROY, + PHASE_COMPACT, + PHASE_COMPACT_MOVE, + PHASE_COMPACT_UPDATE, + PHASE_COMPACT_UPDATE_CELLS, + PHASE_GC_END, + PHASE_MINOR_GC, + PHASE_EVICT_NURSERY, + PHASE_TRACE_HEAP, + PHASE_BARRIER, + PHASE_UNMARK_GRAY, + PHASE_MARK_ROOTS, + PHASE_BUFFER_GRAY_ROOTS, + PHASE_MARK_CCWS, + PHASE_MARK_STACK, + PHASE_MARK_RUNTIME_DATA, + PHASE_MARK_EMBEDDING, + PHASE_MARK_COMPARTMENTS, + PHASE_PURGE_SHAPE_TABLES, + + PHASE_LIMIT, + PHASE_NONE = PHASE_LIMIT, + PHASE_EXPLICIT_SUSPENSION = PHASE_LIMIT, + PHASE_IMPLICIT_SUSPENSION, + PHASE_MULTI_PARENTS +}; + +enum Stat { + STAT_NEW_CHUNK, + STAT_DESTROY_CHUNK, + STAT_MINOR_GC, + + // Number of times a 'put' into a storebuffer overflowed, triggering a + // compaction + STAT_STOREBUFFER_OVERFLOW, + + // Number of arenas relocated by compacting GC. + STAT_ARENA_RELOCATED, + + STAT_LIMIT +}; + +struct ZoneGCStats +{ + /* Number of zones collected in this GC. */ + int collectedZoneCount; + + /* Total number of zones in the Runtime at the start of this GC. */ + int zoneCount; + + /* Number of zones swept in this GC. */ + int sweptZoneCount; + + /* Total number of compartments in all zones collected. */ + int collectedCompartmentCount; + + /* Total number of compartments in the Runtime at the start of this GC. */ + int compartmentCount; + + /* Total number of compartments swept by this GC. */ + int sweptCompartmentCount; + + bool isCollectingAllZones() const { return collectedZoneCount == zoneCount; } + + ZoneGCStats() + : collectedZoneCount(0), zoneCount(0), sweptZoneCount(0), + collectedCompartmentCount(0), compartmentCount(0), sweptCompartmentCount(0) + {} +}; + +#define FOR_EACH_GC_PROFILE_TIME(_) \ + _(BeginCallback, "beginCB", PHASE_GC_BEGIN) \ + _(WaitBgThread, "waitBG", PHASE_WAIT_BACKGROUND_THREAD) \ + _(DiscardCode, "discard", PHASE_MARK_DISCARD_CODE) \ + _(RelazifyFunc, "relazify", PHASE_RELAZIFY_FUNCTIONS) \ + _(PurgeTables, "purgeTables", PHASE_PURGE_SHAPE_TABLES) \ + _(Purge, "purge", PHASE_PURGE) \ + _(Mark, "mark", PHASE_MARK) \ + _(Sweep, "sweep", PHASE_SWEEP) \ + _(Compact, "compact", PHASE_COMPACT) \ + _(EndCallback, "endCB", PHASE_GC_END) \ + _(Barriers, "barriers", PHASE_BARRIER) + +const char* ExplainAbortReason(gc::AbortReason reason); +const char* ExplainInvocationKind(JSGCInvocationKind gckind); + +/* + * Struct for collecting timing statistics on a "phase tree". The tree is + * specified as a limited DAG, but the timings are collected for the whole tree + * that you would get by expanding out the DAG by duplicating subtrees rooted + * at nodes with multiple parents. + * + * During execution, a child phase can be activated multiple times, and the + * total time will be accumulated. (So for example, you can start and end + * PHASE_MARK_ROOTS multiple times before completing the parent phase.) + * + * Incremental GC is represented by recording separate timing results for each + * slice within the overall GC. + */ +struct Statistics +{ + /* + * Phases are allowed to have multiple parents, though any path from root + * to leaf is allowed at most one multi-parented phase. We keep a full set + * of timings for each of the multi-parented phases, to be able to record + * all the timings in the expanded tree induced by our dag. + * + * Note that this wastes quite a bit of space, since we have a whole + * separate array of timing data containing all the phases. We could be + * more clever and keep an array of pointers biased by the offset of the + * multi-parented phase, and thereby preserve the simple + * timings[slot][PHASE_*] indexing. But the complexity doesn't seem worth + * the few hundred bytes of savings. If we want to extend things to full + * DAGs, this decision should be reconsidered. + */ + static const size_t MaxMultiparentPhases = 6; + static const size_t NumTimingArrays = MaxMultiparentPhases + 1; + + /* Create a convenient type for referring to tables of phase times. */ + using PhaseTimeTable = int64_t[NumTimingArrays][PHASE_LIMIT]; + + static MOZ_MUST_USE bool initialize(); + + explicit Statistics(JSRuntime* rt); + ~Statistics(); + + void beginPhase(Phase phase); + void endPhase(Phase phase); + void endParallelPhase(Phase phase, const GCParallelTask* task); + + // Occasionally, we may be in the middle of something that is tracked by + // this class, and we need to do something unusual (eg evict the nursery) + // that doesn't normally nest within the current phase. Suspend the + // currently tracked phase stack, at which time the caller is free to do + // other tracked operations. + // + // This also happens internally with PHASE_GC_BEGIN and other "non-GC" + // phases. While in these phases, any beginPhase will automatically suspend + // the non-GC phase, until that inner stack is complete, at which time it + // will automatically resume the non-GC phase. Explicit suspensions do not + // get auto-resumed. + void suspendPhases(Phase suspension = PHASE_EXPLICIT_SUSPENSION); + + // Resume a suspended stack of phases. + void resumePhases(); + + void beginSlice(const ZoneGCStats& zoneStats, JSGCInvocationKind gckind, + SliceBudget budget, JS::gcreason::Reason reason); + void endSlice(); + + MOZ_MUST_USE bool startTimingMutator(); + MOZ_MUST_USE bool stopTimingMutator(double& mutator_ms, double& gc_ms); + + // Note when we sweep a zone or compartment. + void sweptZone() { ++zoneStats.sweptZoneCount; } + void sweptCompartment() { ++zoneStats.sweptCompartmentCount; } + + void reset(gc::AbortReason reason) { + MOZ_ASSERT(reason != gc::AbortReason::None); + if (!aborted) + slices.back().resetReason = reason; + } + + void nonincremental(gc::AbortReason reason) { + MOZ_ASSERT(reason != gc::AbortReason::None); + nonincrementalReason_ = reason; + } + + bool nonincremental() const { + return nonincrementalReason_ != gc::AbortReason::None; + } + + const char* nonincrementalReason() const { + return ExplainAbortReason(nonincrementalReason_); + } + + void count(Stat s) { + MOZ_ASSERT(s < STAT_LIMIT); + counts[s]++; + } + + void beginNurseryCollection(JS::gcreason::Reason reason); + void endNurseryCollection(JS::gcreason::Reason reason); + + int64_t beginSCC(); + void endSCC(unsigned scc, int64_t start); + + UniqueChars formatCompactSliceMessage() const; + UniqueChars formatCompactSummaryMessage() const; + UniqueChars formatJsonMessage(uint64_t timestamp); + UniqueChars formatDetailedMessage(); + + JS::GCSliceCallback setSliceCallback(JS::GCSliceCallback callback); + JS::GCNurseryCollectionCallback setNurseryCollectionCallback( + JS::GCNurseryCollectionCallback callback); + + int64_t clearMaxGCPauseAccumulator(); + int64_t getMaxGCPauseSinceClear(); + + // Return the current phase, suppressing the synthetic PHASE_MUTATOR phase. + Phase currentPhase() { + if (phaseNestingDepth == 0) + return PHASE_NONE; + if (phaseNestingDepth == 1) + return phaseNesting[0] == PHASE_MUTATOR ? PHASE_NONE : phaseNesting[0]; + return phaseNesting[phaseNestingDepth - 1]; + } + + static const size_t MAX_NESTING = 20; + + struct SliceData { + SliceData(SliceBudget budget, JS::gcreason::Reason reason, int64_t start, + double startTimestamp, size_t startFaults, gc::State initialState) + : budget(budget), reason(reason), + initialState(initialState), + finalState(gc::State::NotActive), + resetReason(gc::AbortReason::None), + start(start), startTimestamp(startTimestamp), + startFaults(startFaults) + { + for (auto i : mozilla::MakeRange(NumTimingArrays)) + mozilla::PodArrayZero(phaseTimes[i]); + } + + SliceBudget budget; + JS::gcreason::Reason reason; + gc::State initialState, finalState; + gc::AbortReason resetReason; + int64_t start, end; + double startTimestamp, endTimestamp; + size_t startFaults, endFaults; + PhaseTimeTable phaseTimes; + + int64_t duration() const { return end - start; } + bool wasReset() const { return resetReason != gc::AbortReason::None; } + }; + + typedef Vector<SliceData, 8, SystemAllocPolicy> SliceDataVector; + typedef SliceDataVector::ConstRange SliceRange; + + SliceRange sliceRange() const { return slices.all(); } + size_t slicesLength() const { return slices.length(); } + + /* Print total profile times on shutdown. */ + void printTotalProfileTimes(); + + private: + JSRuntime* runtime; + + int64_t startupTime; + + /* File pointer used for MOZ_GCTIMER output. */ + FILE* fp; + + /* + * GCs can't really nest, but a second GC can be triggered from within the + * JSGC_END callback. + */ + int gcDepth; + + ZoneGCStats zoneStats; + + JSGCInvocationKind gckind; + + gc::AbortReason nonincrementalReason_; + + SliceDataVector slices; + + /* Most recent time when the given phase started. */ + int64_t phaseStartTimes[PHASE_LIMIT]; + + /* Bookkeeping for GC timings when timingMutator is true */ + int64_t timedGCStart; + int64_t timedGCTime; + + /* Total time in a given phase for this GC. */ + PhaseTimeTable phaseTimes; + + /* Total time in a given phase over all GCs. */ + PhaseTimeTable phaseTotals; + + /* Number of events of this type for this GC. */ + unsigned int counts[STAT_LIMIT]; + + /* Allocated space before the GC started. */ + size_t preBytes; + + /* Records the maximum GC pause in an API-controlled interval (in us). */ + mutable int64_t maxPauseInInterval; + + /* Phases that are currently on stack. */ + Phase phaseNesting[MAX_NESTING]; + size_t phaseNestingDepth; + size_t activeDagSlot; + + /* + * Certain phases can interrupt the phase stack, eg callback phases. When + * this happens, we move the suspended phases over to a sepearate list, + * terminated by a dummy PHASE_SUSPENSION phase (so that we can nest + * suspensions by suspending multiple stacks with a PHASE_SUSPENSION in + * between). + */ + Phase suspendedPhases[MAX_NESTING * 3]; + size_t suspended; + + /* Sweep times for SCCs of compartments. */ + Vector<int64_t, 0, SystemAllocPolicy> sccTimes; + + JS::GCSliceCallback sliceCallback; + JS::GCNurseryCollectionCallback nurseryCollectionCallback; + + /* + * True if we saw an OOM while allocating slices. The statistics for this + * GC will be invalid. + */ + bool aborted; + + /* Profiling data. */ + + enum class ProfileKey + { + Total, +#define DEFINE_TIME_KEY(name, text, phase) \ + name, +FOR_EACH_GC_PROFILE_TIME(DEFINE_TIME_KEY) +#undef DEFINE_TIME_KEY + KeyCount + }; + + using ProfileTimes = mozilla::EnumeratedArray<ProfileKey, ProfileKey::KeyCount, int64_t>; + + int64_t profileThreshold_; + bool enableProfiling_; + ProfileTimes totalTimes_; + uint64_t sliceCount_; + + void beginGC(JSGCInvocationKind kind); + void endGC(); + + void recordPhaseEnd(Phase phase); + + void gcDuration(int64_t* total, int64_t* maxPause) const; + void sccDurations(int64_t* total, int64_t* maxPause); + void printStats(); + + UniqueChars formatCompactSlicePhaseTimes(const PhaseTimeTable phaseTimes) const; + + UniqueChars formatDetailedDescription(); + UniqueChars formatDetailedSliceDescription(unsigned i, const SliceData& slice); + UniqueChars formatDetailedPhaseTimes(const PhaseTimeTable phaseTimes); + UniqueChars formatDetailedTotals(); + + UniqueChars formatJsonDescription(uint64_t timestamp); + UniqueChars formatJsonSliceDescription(unsigned i, const SliceData& slice); + UniqueChars formatJsonPhaseTimes(const PhaseTimeTable phaseTimes); + + double computeMMU(int64_t resolution) const; + + void printSliceProfile(); + static void printProfileHeader(); + static void printProfileTimes(const ProfileTimes& times); +}; + +struct MOZ_RAII AutoGCSlice +{ + AutoGCSlice(Statistics& stats, const ZoneGCStats& zoneStats, JSGCInvocationKind gckind, + SliceBudget budget, JS::gcreason::Reason reason) + : stats(stats) + { + stats.beginSlice(zoneStats, gckind, budget, reason); + } + ~AutoGCSlice() { stats.endSlice(); } + + Statistics& stats; +}; + +struct MOZ_RAII AutoPhase +{ + AutoPhase(Statistics& stats, Phase phase) + : stats(stats), task(nullptr), phase(phase), enabled(true) + { + stats.beginPhase(phase); + } + + AutoPhase(Statistics& stats, bool condition, Phase phase) + : stats(stats), task(nullptr), phase(phase), enabled(condition) + { + if (enabled) + stats.beginPhase(phase); + } + + AutoPhase(Statistics& stats, const GCParallelTask& task, Phase phase) + : stats(stats), task(&task), phase(phase), enabled(true) + { + if (enabled) + stats.beginPhase(phase); + } + + ~AutoPhase() { + if (enabled) { + if (task) + stats.endParallelPhase(phase, task); + else + stats.endPhase(phase); + } + } + + Statistics& stats; + const GCParallelTask* task; + Phase phase; + bool enabled; +}; + +struct MOZ_RAII AutoSCC +{ + AutoSCC(Statistics& stats, unsigned scc) + : stats(stats), scc(scc) + { + start = stats.beginSCC(); + } + ~AutoSCC() { + stats.endSCC(scc, start); + } + + Statistics& stats; + unsigned scc; + int64_t start; +}; + +} /* namespace gcstats */ +} /* namespace js */ + +#endif /* gc_Statistics_h */ diff --git a/js/src/gc/StoreBuffer-inl.h b/js/src/gc/StoreBuffer-inl.h new file mode 100644 index 000000000..5887c52a9 --- /dev/null +++ b/js/src/gc/StoreBuffer-inl.h @@ -0,0 +1,75 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_StoreBuffer_inl_h +#define gc_StoreBuffer_inl_h + +#include "gc/StoreBuffer.h" + +#include "gc/Heap.h" + +namespace js { +namespace gc { + +inline /* static */ size_t +ArenaCellSet::getCellIndex(const TenuredCell* cell) +{ + MOZ_ASSERT((uintptr_t(cell) & ~ArenaMask) % CellSize == 0); + return (uintptr_t(cell) & ArenaMask) / CellSize; +} + +inline /* static */ void +ArenaCellSet::getWordIndexAndMask(size_t cellIndex, size_t* wordp, uint32_t* maskp) +{ + BitArray<ArenaCellCount>::getIndexAndMask(cellIndex, wordp, maskp); +} + +inline bool +ArenaCellSet::hasCell(size_t cellIndex) const +{ + MOZ_ASSERT(cellIndex < ArenaCellCount); + return bits.get(cellIndex); +} + +inline void +ArenaCellSet::putCell(size_t cellIndex) +{ + MOZ_ASSERT(cellIndex < ArenaCellCount); + bits.set(cellIndex); +} + +inline void +ArenaCellSet::check() const +{ +#ifdef DEBUG + bool bitsZero = bits.isAllClear(); + MOZ_ASSERT(isEmpty() == bitsZero); + MOZ_ASSERT(isEmpty() == !arena); + MOZ_ASSERT_IF(!isEmpty(), arena->bufferedCells == this); +#endif +} + +inline void +StoreBuffer::putWholeCell(Cell* cell) +{ + MOZ_ASSERT(cell->isTenured()); + + Arena* arena = cell->asTenured().arena(); + ArenaCellSet* cells = arena->bufferedCells; + if (cells->isEmpty()) { + cells = AllocateWholeCellSet(arena); + if (!cells) + return; + } + + cells->putCell(&cell->asTenured()); + cells->check(); +} + +} // namespace gc +} // namespace js + +#endif // gc_StoreBuffer_inl_h diff --git a/js/src/gc/StoreBuffer.cpp b/js/src/gc/StoreBuffer.cpp new file mode 100644 index 000000000..af13778f0 --- /dev/null +++ b/js/src/gc/StoreBuffer.cpp @@ -0,0 +1,153 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/StoreBuffer-inl.h" + +#include "mozilla/Assertions.h" + +#include "jscompartment.h" + +#include "gc/Statistics.h" +#include "vm/ArgumentsObject.h" +#include "vm/Runtime.h" + +#include "jsgcinlines.h" + +using namespace js; +using namespace js::gc; + +void +StoreBuffer::GenericBuffer::trace(StoreBuffer* owner, JSTracer* trc) +{ + mozilla::ReentrancyGuard g(*owner); + MOZ_ASSERT(owner->isEnabled()); + if (!storage_) + return; + + for (LifoAlloc::Enum e(*storage_); !e.empty();) { + unsigned size = *e.get<unsigned>(); + e.popFront<unsigned>(); + BufferableRef* edge = e.get<BufferableRef>(size); + edge->trace(trc); + e.popFront(size); + } +} + +bool +StoreBuffer::enable() +{ + if (enabled_) + return true; + + if (!bufferVal.init() || + !bufferCell.init() || + !bufferSlot.init() || + !bufferGeneric.init()) + { + return false; + } + + enabled_ = true; + return true; +} + +void +StoreBuffer::disable() +{ + if (!enabled_) + return; + + aboutToOverflow_ = false; + + enabled_ = false; +} + +void +StoreBuffer::clear() +{ + if (!enabled_) + return; + + aboutToOverflow_ = false; + cancelIonCompilations_ = false; + + bufferVal.clear(); + bufferCell.clear(); + bufferSlot.clear(); + bufferGeneric.clear(); + + for (ArenaCellSet* set = bufferWholeCell; set; set = set->next) + set->arena->bufferedCells = nullptr; + bufferWholeCell = nullptr; +} + +void +StoreBuffer::setAboutToOverflow() +{ + if (!aboutToOverflow_) { + aboutToOverflow_ = true; + runtime_->gc.stats.count(gcstats::STAT_STOREBUFFER_OVERFLOW); + } + runtime_->gc.requestMinorGC(JS::gcreason::FULL_STORE_BUFFER); +} + +void +StoreBuffer::addSizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf, JS::GCSizes +*sizes) +{ + sizes->storeBufferVals += bufferVal.sizeOfExcludingThis(mallocSizeOf); + sizes->storeBufferCells += bufferCell.sizeOfExcludingThis(mallocSizeOf); + sizes->storeBufferSlots += bufferSlot.sizeOfExcludingThis(mallocSizeOf); + sizes->storeBufferGenerics += bufferGeneric.sizeOfExcludingThis(mallocSizeOf); + + for (ArenaCellSet* set = bufferWholeCell; set; set = set->next) + sizes->storeBufferWholeCells += sizeof(ArenaCellSet); +} + +void +StoreBuffer::addToWholeCellBuffer(ArenaCellSet* set) +{ + set->next = bufferWholeCell; + bufferWholeCell = set; +} + +ArenaCellSet ArenaCellSet::Empty(nullptr); + +ArenaCellSet::ArenaCellSet(Arena* arena) + : arena(arena), next(nullptr) +{ + bits.clear(false); +} + +ArenaCellSet* +js::gc::AllocateWholeCellSet(Arena* arena) +{ + Zone* zone = arena->zone; + JSRuntime* rt = zone->runtimeFromMainThread(); + if (!rt->gc.nursery.isEnabled()) + return nullptr; + + AutoEnterOOMUnsafeRegion oomUnsafe; + Nursery& nursery = rt->gc.nursery; + void* data = nursery.allocateBuffer(zone, sizeof(ArenaCellSet)); + if (!data) { + oomUnsafe.crash("Failed to allocate WholeCellSet"); + return nullptr; + } + + if (nursery.freeSpace() < ArenaCellSet::NurseryFreeThresholdBytes) + rt->gc.storeBuffer.setAboutToOverflow(); + + auto cells = static_cast<ArenaCellSet*>(data); + new (cells) ArenaCellSet(arena); + arena->bufferedCells = cells; + rt->gc.storeBuffer.addToWholeCellBuffer(cells); + return cells; +} + +template struct StoreBuffer::MonoTypeBuffer<StoreBuffer::ValueEdge>; +template struct StoreBuffer::MonoTypeBuffer<StoreBuffer::CellPtrEdge>; +template struct StoreBuffer::MonoTypeBuffer<StoreBuffer::SlotsEdge>; diff --git a/js/src/gc/StoreBuffer.h b/js/src/gc/StoreBuffer.h new file mode 100644 index 000000000..f8d9031f1 --- /dev/null +++ b/js/src/gc/StoreBuffer.h @@ -0,0 +1,499 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_StoreBuffer_h +#define gc_StoreBuffer_h + +#include "mozilla/Attributes.h" +#include "mozilla/ReentrancyGuard.h" + +#include <algorithm> + +#include "jsalloc.h" + +#include "ds/LifoAlloc.h" +#include "gc/Nursery.h" +#include "js/MemoryMetrics.h" + +namespace js { +namespace gc { + +class ArenaCellSet; + +/* + * BufferableRef represents an abstract reference for use in the generational + * GC's remembered set. Entries in the store buffer that cannot be represented + * with the simple pointer-to-a-pointer scheme must derive from this class and + * use the generic store buffer interface. + * + * A single BufferableRef entry in the generic buffer can represent many entries + * in the remembered set. For example js::OrderedHashTableRef represents all + * the incoming edges corresponding to keys in an ordered hash table. + */ +class BufferableRef +{ + public: + virtual void trace(JSTracer* trc) = 0; + bool maybeInRememberedSet(const Nursery&) const { return true; } +}; + +typedef HashSet<void*, PointerHasher<void*, 3>, SystemAllocPolicy> EdgeSet; + +/* The size of a single block of store buffer storage space. */ +static const size_t LifoAllocBlockSize = 1 << 13; /* 8KiB */ + +/* + * The StoreBuffer observes all writes that occur in the system and performs + * efficient filtering of them to derive a remembered set for nursery GC. + */ +class StoreBuffer +{ + friend class mozilla::ReentrancyGuard; + + /* The size at which a block is about to overflow. */ + static const size_t LowAvailableThreshold = size_t(LifoAllocBlockSize / 2.0); + + /* + * This buffer holds only a single type of edge. Using this buffer is more + * efficient than the generic buffer when many writes will be to the same + * type of edge: e.g. Value or Cell*. + */ + template<typename T> + struct MonoTypeBuffer + { + /* The canonical set of stores. */ + typedef HashSet<T, typename T::Hasher, SystemAllocPolicy> StoreSet; + StoreSet stores_; + + /* + * A one element cache in front of the canonical set to speed up + * temporary instances of HeapPtr. + */ + T last_; + + /* Maximum number of entries before we request a minor GC. */ + const static size_t MaxEntries = 48 * 1024 / sizeof(T); + + explicit MonoTypeBuffer() : last_(T()) {} + ~MonoTypeBuffer() { stores_.finish(); } + + MOZ_MUST_USE bool init() { + if (!stores_.initialized() && !stores_.init()) + return false; + clear(); + return true; + } + + void clear() { + last_ = T(); + if (stores_.initialized()) + stores_.clear(); + } + + /* Add one item to the buffer. */ + void put(StoreBuffer* owner, const T& t) { + MOZ_ASSERT(stores_.initialized()); + sinkStore(owner); + last_ = t; + } + + /* Remove an item from the store buffer. */ + void unput(StoreBuffer* owner, const T& v) { + // Fast, hashless remove of last put. + if (last_ == v) { + last_ = T(); + return; + } + stores_.remove(v); + } + + /* Move any buffered stores to the canonical store set. */ + void sinkStore(StoreBuffer* owner) { + MOZ_ASSERT(stores_.initialized()); + if (last_) { + AutoEnterOOMUnsafeRegion oomUnsafe; + if (!stores_.put(last_)) + oomUnsafe.crash("Failed to allocate for MonoTypeBuffer::put."); + } + last_ = T(); + + if (MOZ_UNLIKELY(stores_.count() > MaxEntries)) + owner->setAboutToOverflow(); + } + + bool has(StoreBuffer* owner, const T& v) { + sinkStore(owner); + return stores_.has(v); + } + + /* Trace the source of all edges in the store buffer. */ + void trace(StoreBuffer* owner, TenuringTracer& mover); + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) { + return stores_.sizeOfExcludingThis(mallocSizeOf); + } + + private: + MonoTypeBuffer& operator=(const MonoTypeBuffer& other) = delete; + }; + + struct GenericBuffer + { + LifoAlloc* storage_; + + explicit GenericBuffer() : storage_(nullptr) {} + ~GenericBuffer() { js_delete(storage_); } + + MOZ_MUST_USE bool init() { + if (!storage_) + storage_ = js_new<LifoAlloc>(LifoAllocBlockSize); + clear(); + return bool(storage_); + } + + void clear() { + if (!storage_) + return; + + storage_->used() ? storage_->releaseAll() : storage_->freeAll(); + } + + bool isAboutToOverflow() const { + return !storage_->isEmpty() && + storage_->availableInCurrentChunk() < LowAvailableThreshold; + } + + /* Trace all generic edges. */ + void trace(StoreBuffer* owner, JSTracer* trc); + + template <typename T> + void put(StoreBuffer* owner, const T& t) { + MOZ_ASSERT(storage_); + + /* Ensure T is derived from BufferableRef. */ + (void)static_cast<const BufferableRef*>(&t); + + AutoEnterOOMUnsafeRegion oomUnsafe; + unsigned size = sizeof(T); + unsigned* sizep = storage_->pod_malloc<unsigned>(); + if (!sizep) + oomUnsafe.crash("Failed to allocate for GenericBuffer::put."); + *sizep = size; + + T* tp = storage_->new_<T>(t); + if (!tp) + oomUnsafe.crash("Failed to allocate for GenericBuffer::put."); + + if (isAboutToOverflow()) + owner->setAboutToOverflow(); + } + + size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) { + return storage_ ? storage_->sizeOfIncludingThis(mallocSizeOf) : 0; + } + + bool isEmpty() { + return !storage_ || storage_->isEmpty(); + } + + private: + GenericBuffer& operator=(const GenericBuffer& other) = delete; + }; + + template <typename Edge> + struct PointerEdgeHasher + { + typedef Edge Lookup; + static HashNumber hash(const Lookup& l) { return uintptr_t(l.edge) >> 3; } + static bool match(const Edge& k, const Lookup& l) { return k == l; } + }; + + struct CellPtrEdge + { + Cell** edge; + + CellPtrEdge() : edge(nullptr) {} + explicit CellPtrEdge(Cell** v) : edge(v) {} + bool operator==(const CellPtrEdge& other) const { return edge == other.edge; } + bool operator!=(const CellPtrEdge& other) const { return edge != other.edge; } + + bool maybeInRememberedSet(const Nursery& nursery) const { + MOZ_ASSERT(IsInsideNursery(*edge)); + return !nursery.isInside(edge); + } + + void trace(TenuringTracer& mover) const; + + CellPtrEdge tagged() const { return CellPtrEdge((Cell**)(uintptr_t(edge) | 1)); } + CellPtrEdge untagged() const { return CellPtrEdge((Cell**)(uintptr_t(edge) & ~1)); } + bool isTagged() const { return bool(uintptr_t(edge) & 1); } + + explicit operator bool() const { return edge != nullptr; } + + typedef PointerEdgeHasher<CellPtrEdge> Hasher; + }; + + struct ValueEdge + { + JS::Value* edge; + + ValueEdge() : edge(nullptr) {} + explicit ValueEdge(JS::Value* v) : edge(v) {} + bool operator==(const ValueEdge& other) const { return edge == other.edge; } + bool operator!=(const ValueEdge& other) const { return edge != other.edge; } + + Cell* deref() const { return edge->isGCThing() ? static_cast<Cell*>(edge->toGCThing()) : nullptr; } + + bool maybeInRememberedSet(const Nursery& nursery) const { + MOZ_ASSERT(IsInsideNursery(deref())); + return !nursery.isInside(edge); + } + + void trace(TenuringTracer& mover) const; + + ValueEdge tagged() const { return ValueEdge((JS::Value*)(uintptr_t(edge) | 1)); } + ValueEdge untagged() const { return ValueEdge((JS::Value*)(uintptr_t(edge) & ~1)); } + bool isTagged() const { return bool(uintptr_t(edge) & 1); } + + explicit operator bool() const { return edge != nullptr; } + + typedef PointerEdgeHasher<ValueEdge> Hasher; + }; + + struct SlotsEdge + { + // These definitions must match those in HeapSlot::Kind. + const static int SlotKind = 0; + const static int ElementKind = 1; + + uintptr_t objectAndKind_; // NativeObject* | Kind + int32_t start_; + int32_t count_; + + SlotsEdge() : objectAndKind_(0), start_(0), count_(0) {} + SlotsEdge(NativeObject* object, int kind, int32_t start, int32_t count) + : objectAndKind_(uintptr_t(object) | kind), start_(start), count_(count) + { + MOZ_ASSERT((uintptr_t(object) & 1) == 0); + MOZ_ASSERT(kind <= 1); + MOZ_ASSERT(start >= 0); + MOZ_ASSERT(count > 0); + } + + NativeObject* object() const { return reinterpret_cast<NativeObject*>(objectAndKind_ & ~1); } + int kind() const { return (int)(objectAndKind_ & 1); } + + bool operator==(const SlotsEdge& other) const { + return objectAndKind_ == other.objectAndKind_ && + start_ == other.start_ && + count_ == other.count_; + } + + bool operator!=(const SlotsEdge& other) const { + return !(*this == other); + } + + // True if this SlotsEdge range overlaps with the other SlotsEdge range, + // false if they do not overlap. + bool overlaps(const SlotsEdge& other) const { + if (objectAndKind_ != other.objectAndKind_) + return false; + + // Widen our range by one on each side so that we consider + // adjacent-but-not-actually-overlapping ranges as overlapping. This + // is particularly useful for coalescing a series of increasing or + // decreasing single index writes 0, 1, 2, ..., N into a SlotsEdge + // range of elements [0, N]. + auto end = start_ + count_ + 1; + auto start = start_ - 1; + + auto otherEnd = other.start_ + other.count_; + return (start <= other.start_ && other.start_ <= end) || + (start <= otherEnd && otherEnd <= end); + } + + // Destructively make this SlotsEdge range the union of the other + // SlotsEdge range and this one. A precondition is that the ranges must + // overlap. + void merge(const SlotsEdge& other) { + MOZ_ASSERT(overlaps(other)); + auto end = Max(start_ + count_, other.start_ + other.count_); + start_ = Min(start_, other.start_); + count_ = end - start_; + } + + bool maybeInRememberedSet(const Nursery& n) const { + return !IsInsideNursery(reinterpret_cast<Cell*>(object())); + } + + void trace(TenuringTracer& mover) const; + + explicit operator bool() const { return objectAndKind_ != 0; } + + typedef struct { + typedef SlotsEdge Lookup; + static HashNumber hash(const Lookup& l) { return l.objectAndKind_ ^ l.start_ ^ l.count_; } + static bool match(const SlotsEdge& k, const Lookup& l) { return k == l; } + } Hasher; + }; + + template <typename Buffer, typename Edge> + void unput(Buffer& buffer, const Edge& edge) { + MOZ_ASSERT(!JS::shadow::Runtime::asShadowRuntime(runtime_)->isHeapBusy()); + MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime_)); + if (!isEnabled()) + return; + mozilla::ReentrancyGuard g(*this); + buffer.unput(this, edge); + } + + template <typename Buffer, typename Edge> + void put(Buffer& buffer, const Edge& edge) { + MOZ_ASSERT(!JS::shadow::Runtime::asShadowRuntime(runtime_)->isHeapBusy()); + MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtime_)); + if (!isEnabled()) + return; + mozilla::ReentrancyGuard g(*this); + if (edge.maybeInRememberedSet(nursery_)) + buffer.put(this, edge); + } + + MonoTypeBuffer<ValueEdge> bufferVal; + MonoTypeBuffer<CellPtrEdge> bufferCell; + MonoTypeBuffer<SlotsEdge> bufferSlot; + ArenaCellSet* bufferWholeCell; + GenericBuffer bufferGeneric; + bool cancelIonCompilations_; + + JSRuntime* runtime_; + const Nursery& nursery_; + + bool aboutToOverflow_; + bool enabled_; +#ifdef DEBUG + bool mEntered; /* For ReentrancyGuard. */ +#endif + + public: + explicit StoreBuffer(JSRuntime* rt, const Nursery& nursery) + : bufferVal(), bufferCell(), bufferSlot(), bufferWholeCell(nullptr), bufferGeneric(), + cancelIonCompilations_(false), runtime_(rt), nursery_(nursery), aboutToOverflow_(false), + enabled_(false) +#ifdef DEBUG + , mEntered(false) +#endif + { + } + + MOZ_MUST_USE bool enable(); + void disable(); + bool isEnabled() const { return enabled_; } + + void clear(); + + /* Get the overflowed status. */ + bool isAboutToOverflow() const { return aboutToOverflow_; } + + bool cancelIonCompilations() const { return cancelIonCompilations_; } + + /* Insert a single edge into the buffer/remembered set. */ + void putValue(JS::Value* vp) { put(bufferVal, ValueEdge(vp)); } + void unputValue(JS::Value* vp) { unput(bufferVal, ValueEdge(vp)); } + void putCell(Cell** cellp) { put(bufferCell, CellPtrEdge(cellp)); } + void unputCell(Cell** cellp) { unput(bufferCell, CellPtrEdge(cellp)); } + void putSlot(NativeObject* obj, int kind, int32_t start, int32_t count) { + SlotsEdge edge(obj, kind, start, count); + if (bufferSlot.last_.overlaps(edge)) + bufferSlot.last_.merge(edge); + else + put(bufferSlot, edge); + } + inline void putWholeCell(Cell* cell); + + /* Insert an entry into the generic buffer. */ + template <typename T> + void putGeneric(const T& t) { put(bufferGeneric, t);} + + void setShouldCancelIonCompilations() { + cancelIonCompilations_ = true; + } + + /* Methods to trace the source of all edges in the store buffer. */ + void traceValues(TenuringTracer& mover) { bufferVal.trace(this, mover); } + void traceCells(TenuringTracer& mover) { bufferCell.trace(this, mover); } + void traceSlots(TenuringTracer& mover) { bufferSlot.trace(this, mover); } + void traceGenericEntries(JSTracer *trc) { bufferGeneric.trace(this, trc); } + + void traceWholeCells(TenuringTracer& mover); + void traceWholeCell(TenuringTracer& mover, JS::TraceKind kind, Cell* cell); + + /* For use by our owned buffers and for testing. */ + void setAboutToOverflow(); + + void addToWholeCellBuffer(ArenaCellSet* set); + + void addSizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf, JS::GCSizes* sizes); +}; + +// A set of cells in an arena used to implement the whole cell store buffer. +class ArenaCellSet +{ + friend class StoreBuffer; + + // The arena this relates to. + Arena* arena; + + // Pointer to next set forming a linked list. + ArenaCellSet* next; + + // Bit vector for each possible cell start position. + BitArray<ArenaCellCount> bits; + + public: + explicit ArenaCellSet(Arena* arena); + + bool hasCell(const TenuredCell* cell) const { + return hasCell(getCellIndex(cell)); + } + + void putCell(const TenuredCell* cell) { + putCell(getCellIndex(cell)); + } + + bool isEmpty() const { + return this == &Empty; + } + + bool hasCell(size_t cellIndex) const; + + void putCell(size_t cellIndex); + + void check() const; + + // Sentinel object used for all empty sets. + static ArenaCellSet Empty; + + static size_t getCellIndex(const TenuredCell* cell); + static void getWordIndexAndMask(size_t cellIndex, size_t* wordp, uint32_t* maskp); + + // Attempt to trigger a minor GC if free space in the nursery (where these + // objects are allocated) falls below this threshold. + static const size_t NurseryFreeThresholdBytes = 64 * 1024; + + static size_t offsetOfArena() { + return offsetof(ArenaCellSet, arena); + } + static size_t offsetOfBits() { + return offsetof(ArenaCellSet, bits); + } +}; + +ArenaCellSet* AllocateWholeCellSet(Arena* arena); + +} /* namespace gc */ +} /* namespace js */ + +#endif /* gc_StoreBuffer_h */ diff --git a/js/src/gc/Tracer.cpp b/js/src/gc/Tracer.cpp new file mode 100644 index 000000000..63cd9b08a --- /dev/null +++ b/js/src/gc/Tracer.cpp @@ -0,0 +1,432 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/Tracer.h" + +#include "mozilla/DebugOnly.h" +#include "mozilla/SizePrintfMacros.h" + +#include "jsapi.h" +#include "jsfun.h" +#include "jsgc.h" +#include "jsprf.h" +#include "jsscript.h" +#include "jsutil.h" +#include "NamespaceImports.h" + +#include "gc/GCInternals.h" +#include "gc/Marking.h" +#include "gc/Zone.h" + +#include "vm/Shape.h" +#include "vm/Symbol.h" + +#include "jscompartmentinlines.h" +#include "jsgcinlines.h" + +#include "vm/ObjectGroup-inl.h" + +using namespace js; +using namespace js::gc; +using mozilla::DebugOnly; + +namespace js { +template<typename T> +void +CheckTracedThing(JSTracer* trc, T thing); +} // namespace js + + +/*** Callback Tracer Dispatch ********************************************************************/ + +template <typename T> +T +DoCallback(JS::CallbackTracer* trc, T* thingp, const char* name) +{ + CheckTracedThing(trc, *thingp); + JS::AutoTracingName ctx(trc, name); + trc->dispatchToOnEdge(thingp); + return *thingp; +} +#define INSTANTIATE_ALL_VALID_TRACE_FUNCTIONS(name, type, _) \ + template type* DoCallback<type*>(JS::CallbackTracer*, type**, const char*); +JS_FOR_EACH_TRACEKIND(INSTANTIATE_ALL_VALID_TRACE_FUNCTIONS); +#undef INSTANTIATE_ALL_VALID_TRACE_FUNCTIONS + +template <typename S> +struct DoCallbackFunctor : public IdentityDefaultAdaptor<S> { + template <typename T> S operator()(T* t, JS::CallbackTracer* trc, const char* name) { + return js::gc::RewrapTaggedPointer<S, T>::wrap(DoCallback(trc, &t, name)); + } +}; + +template <> +Value +DoCallback<Value>(JS::CallbackTracer* trc, Value* vp, const char* name) +{ + *vp = DispatchTyped(DoCallbackFunctor<Value>(), *vp, trc, name); + return *vp; +} + +template <> +jsid +DoCallback<jsid>(JS::CallbackTracer* trc, jsid* idp, const char* name) +{ + *idp = DispatchTyped(DoCallbackFunctor<jsid>(), *idp, trc, name); + return *idp; +} + +template <> +TaggedProto +DoCallback<TaggedProto>(JS::CallbackTracer* trc, TaggedProto* protop, const char* name) +{ + *protop = DispatchTyped(DoCallbackFunctor<TaggedProto>(), *protop, trc, name); + return *protop; +} + +void +JS::CallbackTracer::getTracingEdgeName(char* buffer, size_t bufferSize) +{ + MOZ_ASSERT(bufferSize > 0); + if (contextFunctor_) { + (*contextFunctor_)(this, buffer, bufferSize); + return; + } + if (contextIndex_ != InvalidIndex) { + snprintf(buffer, bufferSize, "%s[%" PRIuSIZE "]", contextName_, contextIndex_); + return; + } + snprintf(buffer, bufferSize, "%s", contextName_); +} + + +/*** Public Tracing API **************************************************************************/ + +JS_PUBLIC_API(void) +JS::TraceChildren(JSTracer* trc, GCCellPtr thing) +{ + js::TraceChildren(trc, thing.asCell(), thing.kind()); +} + +struct TraceChildrenFunctor { + template <typename T> + void operator()(JSTracer* trc, void* thing) { + static_cast<T*>(thing)->traceChildren(trc); + } +}; + +void +js::TraceChildren(JSTracer* trc, void* thing, JS::TraceKind kind) +{ + MOZ_ASSERT(thing); + TraceChildrenFunctor f; + DispatchTraceKindTyped(f, kind, trc, thing); +} + +namespace { +struct TraceIncomingFunctor { + JSTracer* trc_; + const JS::CompartmentSet& compartments_; + TraceIncomingFunctor(JSTracer* trc, const JS::CompartmentSet& compartments) + : trc_(trc), compartments_(compartments) + {} + template <typename T> + void operator()(T tp) { + if (!compartments_.has((*tp)->compartment())) + return; + TraceManuallyBarrieredEdge(trc_, tp, "cross-compartment wrapper"); + } + // StringWrappers are just used to avoid copying strings + // across zones multiple times, and don't hold a strong + // reference. + void operator()(JSString** tp) {} +}; +} // namespace (anonymous) + +JS_PUBLIC_API(void) +JS::TraceIncomingCCWs(JSTracer* trc, const JS::CompartmentSet& compartments) +{ + for (js::CompartmentsIter comp(trc->runtime(), SkipAtoms); !comp.done(); comp.next()) { + if (compartments.has(comp)) + continue; + + for (JSCompartment::WrapperEnum e(comp); !e.empty(); e.popFront()) { + mozilla::DebugOnly<const CrossCompartmentKey> prior = e.front().key(); + e.front().mutableKey().applyToWrapped(TraceIncomingFunctor(trc, compartments)); + MOZ_ASSERT(e.front().key() == prior); + } + } +} + + +/*** Cycle Collector Helpers **********************************************************************/ + +// This function is used by the Cycle Collector (CC) to trace through -- or in +// CC parlance, traverse -- a Shape tree. The CC does not care about Shapes or +// BaseShapes, only the JSObjects held live by them. Thus, we walk the Shape +// lineage, but only report non-Shape things. This effectively makes the entire +// shape lineage into a single node in the CC, saving tremendous amounts of +// space and time in its algorithms. +// +// The algorithm implemented here uses only bounded stack space. This would be +// possible to implement outside the engine, but would require much extra +// infrastructure and many, many more slow GOT lookups. We have implemented it +// inside SpiderMonkey, despite the lack of general applicability, for the +// simplicity and performance of FireFox's embedding of this engine. +void +gc::TraceCycleCollectorChildren(JS::CallbackTracer* trc, Shape* shape) +{ + do { + MOZ_ASSERT(shape->base()); + shape->base()->assertConsistency(); + + TraceEdge(trc, &shape->propidRef(), "propid"); + + if (shape->hasGetterObject()) { + JSObject* tmp = shape->getterObject(); + DoCallback(trc, &tmp, "getter"); + MOZ_ASSERT(tmp == shape->getterObject()); + } + + if (shape->hasSetterObject()) { + JSObject* tmp = shape->setterObject(); + DoCallback(trc, &tmp, "setter"); + MOZ_ASSERT(tmp == shape->setterObject()); + } + + shape = shape->previous(); + } while (shape); +} + +// Object groups can point to other object groups via an UnboxedLayout or the +// the original unboxed group link. There can potentially be deep or cyclic +// chains of such groups to trace through without going through a thing that +// participates in cycle collection. These need to be handled iteratively to +// avoid blowing the stack when running the cycle collector's callback tracer. +struct ObjectGroupCycleCollectorTracer : public JS::CallbackTracer +{ + explicit ObjectGroupCycleCollectorTracer(JS::CallbackTracer* innerTracer) + : JS::CallbackTracer(innerTracer->runtime(), DoNotTraceWeakMaps), + innerTracer(innerTracer) + {} + + void onChild(const JS::GCCellPtr& thing) override; + + JS::CallbackTracer* innerTracer; + Vector<ObjectGroup*, 4, SystemAllocPolicy> seen, worklist; +}; + +void +ObjectGroupCycleCollectorTracer::onChild(const JS::GCCellPtr& thing) +{ + if (thing.is<BaseShape>()) { + // The CC does not care about BaseShapes, and no additional GC things + // will be reached by following this edge. + return; + } + + if (thing.is<JSObject>() || thing.is<JSScript>()) { + // Invoke the inner cycle collector callback on this child. It will not + // recurse back into TraceChildren. + innerTracer->onChild(thing); + return; + } + + if (thing.is<ObjectGroup>()) { + // If this group is required to be in an ObjectGroup chain, trace it + // via the provided worklist rather than continuing to recurse. + ObjectGroup& group = thing.as<ObjectGroup>(); + if (group.maybeUnboxedLayout()) { + for (size_t i = 0; i < seen.length(); i++) { + if (seen[i] == &group) + return; + } + if (seen.append(&group) && worklist.append(&group)) { + return; + } else { + // If append fails, keep tracing normally. The worst that will + // happen is we end up overrecursing. + } + } + } + + TraceChildren(this, thing.asCell(), thing.kind()); +} + +void +gc::TraceCycleCollectorChildren(JS::CallbackTracer* trc, ObjectGroup* group) +{ + MOZ_ASSERT(trc->isCallbackTracer()); + + // Early return if this group is not required to be in an ObjectGroup chain. + if (!group->maybeUnboxedLayout()) + return group->traceChildren(trc); + + ObjectGroupCycleCollectorTracer groupTracer(trc->asCallbackTracer()); + group->traceChildren(&groupTracer); + + while (!groupTracer.worklist.empty()) { + ObjectGroup* innerGroup = groupTracer.worklist.popCopy(); + innerGroup->traceChildren(&groupTracer); + } +} + + +/*** Traced Edge Printer *************************************************************************/ + +static size_t +CountDecimalDigits(size_t num) +{ + size_t numDigits = 0; + do { + num /= 10; + numDigits++; + } while (num > 0); + + return numDigits; +} + +JS_PUBLIC_API(void) +JS_GetTraceThingInfo(char* buf, size_t bufsize, JSTracer* trc, void* thing, + JS::TraceKind kind, bool details) +{ + const char* name = nullptr; /* silence uninitialized warning */ + size_t n; + + if (bufsize == 0) + return; + + switch (kind) { + case JS::TraceKind::Object: + { + name = static_cast<JSObject*>(thing)->getClass()->name; + break; + } + + case JS::TraceKind::Script: + name = "script"; + break; + + case JS::TraceKind::String: + name = ((JSString*)thing)->isDependent() + ? "substring" + : "string"; + break; + + case JS::TraceKind::Symbol: + name = "symbol"; + break; + + case JS::TraceKind::BaseShape: + name = "base_shape"; + break; + + case JS::TraceKind::JitCode: + name = "jitcode"; + break; + + case JS::TraceKind::LazyScript: + name = "lazyscript"; + break; + + case JS::TraceKind::Shape: + name = "shape"; + break; + + case JS::TraceKind::ObjectGroup: + name = "object_group"; + break; + + default: + name = "INVALID"; + break; + } + + n = strlen(name); + if (n > bufsize - 1) + n = bufsize - 1; + js_memcpy(buf, name, n + 1); + buf += n; + bufsize -= n; + *buf = '\0'; + + if (details && bufsize > 2) { + switch (kind) { + case JS::TraceKind::Object: + { + JSObject* obj = (JSObject*)thing; + if (obj->is<JSFunction>()) { + JSFunction* fun = &obj->as<JSFunction>(); + if (fun->displayAtom()) { + *buf++ = ' '; + bufsize--; + PutEscapedString(buf, bufsize, fun->displayAtom(), 0); + } + } else if (obj->getClass()->flags & JSCLASS_HAS_PRIVATE) { + snprintf(buf, bufsize, " %p", obj->as<NativeObject>().getPrivate()); + } else { + snprintf(buf, bufsize, " <no private>"); + } + break; + } + + case JS::TraceKind::Script: + { + JSScript* script = static_cast<JSScript*>(thing); + snprintf(buf, bufsize, " %s:%" PRIuSIZE, script->filename(), script->lineno()); + break; + } + + case JS::TraceKind::String: + { + *buf++ = ' '; + bufsize--; + JSString* str = (JSString*)thing; + + if (str->isLinear()) { + bool willFit = str->length() + strlen("<length > ") + + CountDecimalDigits(str->length()) < bufsize; + + n = snprintf(buf, bufsize, "<length %" PRIuSIZE "%s> ", + str->length(), + willFit ? "" : " (truncated)"); + buf += n; + bufsize -= n; + + PutEscapedString(buf, bufsize, &str->asLinear(), 0); + } else { + snprintf(buf, bufsize, "<rope: length %" PRIuSIZE ">", str->length()); + } + break; + } + + case JS::TraceKind::Symbol: + { + JS::Symbol* sym = static_cast<JS::Symbol*>(thing); + if (JSString* desc = sym->description()) { + if (desc->isLinear()) { + *buf++ = ' '; + bufsize--; + PutEscapedString(buf, bufsize, &desc->asLinear(), 0); + } else { + snprintf(buf, bufsize, "<nonlinear desc>"); + } + } else { + snprintf(buf, bufsize, "<null>"); + } + break; + } + + default: + break; + } + } + buf[bufsize - 1] = '\0'; +} + +JS::CallbackTracer::CallbackTracer(JSContext* cx, WeakMapTraceKind weakTraceKind) + : CallbackTracer(cx->runtime(), weakTraceKind) +{} diff --git a/js/src/gc/Tracer.h b/js/src/gc/Tracer.h new file mode 100644 index 000000000..05c0a4e2e --- /dev/null +++ b/js/src/gc/Tracer.h @@ -0,0 +1,159 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef js_Tracer_h +#define js_Tracer_h + +#include "jsfriendapi.h" + +#include "gc/Barrier.h" + +namespace js { + +// Internal Tracing API +// +// Tracing is an abstract visitation of each edge in a JS heap graph.[1] The +// most common (and performance sensitive) use of this infrastructure is for GC +// "marking" as part of the mark-and-sweep collector; however, this +// infrastructure is much more general than that and is used for many other +// purposes as well. +// +// One commonly misunderstood subtlety of the tracing architecture is the role +// of graph vertices versus graph edges. Graph vertices are the heap +// allocations -- GC things -- that are returned by Allocate. Graph edges are +// pointers -- including tagged pointers like Value and jsid -- that link the +// allocations into a complex heap. The tracing API deals *only* with edges. +// Any action taken on the target of a graph edge is independent of the tracing +// itself. +// +// Another common misunderstanding relates to the role of the JSTracer. The +// JSTracer instance determines what tracing does when visiting an edge; it +// does not itself participate in the tracing process, other than to be passed +// through as opaque data. It works like a closure in that respect. +// +// Tracing implementations internal to SpiderMonkey should use these interfaces +// instead of the public interfaces in js/TracingAPI.h. Unlike the public +// tracing methods, these work on internal types and avoid an external call. +// +// Note that the implementations for these methods are, surprisingly, in +// js/src/gc/Marking.cpp. This is so that the compiler can inline as much as +// possible in the common, marking pathways. Conceptually, however, they remain +// as part of the generic "tracing" architecture, rather than the more specific +// marking implementation of tracing. +// +// 1 - In SpiderMonkey, we call this concept tracing rather than visiting +// because "visiting" is already used by the compiler. Also, it's been +// called "tracing" forever and changing it would be extremely difficult at +// this point. + +// Trace through an edge in the live object graph on behalf of tracing. The +// effect of tracing the edge depends on the JSTracer being used. For pointer +// types, |*thingp| must not be null. +template <typename T> +void +TraceEdge(JSTracer* trc, WriteBarrieredBase<T>* thingp, const char* name); + +template <typename T> +void +TraceEdge(JSTracer* trc, ReadBarriered<T>* thingp, const char* name); + +// Trace through an edge in the live object graph on behalf of tracing. +template <typename T> +void +TraceNullableEdge(JSTracer* trc, WriteBarrieredBase<T>* thingp, const char* name); + +// Trace through a "root" edge. These edges are the initial edges in the object +// graph traversal. Root edges are asserted to only be traversed in the initial +// phase of a GC. +template <typename T> +void +TraceRoot(JSTracer* trc, T* thingp, const char* name); + +template <typename T> +void +TraceRoot(JSTracer* trc, ReadBarriered<T>* thingp, const char* name); + +// Idential to TraceRoot, except that this variant will not crash if |*thingp| +// is null. +template <typename T> +void +TraceNullableRoot(JSTracer* trc, T* thingp, const char* name); + +template <typename T> +void +TraceNullableRoot(JSTracer* trc, ReadBarriered<T>* thingp, const char* name); + +// Like TraceEdge, but for edges that do not use one of the automatic barrier +// classes and, thus, must be treated specially for moving GC. This method is +// separate from TraceEdge to make accidental use of such edges more obvious. +template <typename T> +void +TraceManuallyBarrieredEdge(JSTracer* trc, T* thingp, const char* name); + +// Visits a WeakRef, but does not trace its referents. If *thingp is not marked +// at the end of marking, it is replaced by nullptr. This method records +// thingp, so the edge location must not change after this function is called. +template <typename T> +void +TraceWeakEdge(JSTracer* trc, WeakRef<T>* thingp, const char* name); + +// Trace all edges contained in the given array. +template <typename T> +void +TraceRange(JSTracer* trc, size_t len, WriteBarrieredBase<T>* vec, const char* name); + +// Trace all root edges in the given array. +template <typename T> +void +TraceRootRange(JSTracer* trc, size_t len, T* vec, const char* name); + +// Trace an edge that crosses compartment boundaries. If the compartment of the +// destination thing is not being GC'd, then the edge will not be traced. +template <typename T> +void +TraceCrossCompartmentEdge(JSTracer* trc, JSObject* src, WriteBarrieredBase<T>* dst, + const char* name); + +// As above but with manual barriers. +template <typename T> +void +TraceManuallyBarrieredCrossCompartmentEdge(JSTracer* trc, JSObject* src, T* dst, + const char* name); + +// Permanent atoms and well-known symbols are shared between runtimes and must +// use a separate marking path so that we can filter them out of normal heap +// tracing. +template <typename T> +void +TraceProcessGlobalRoot(JSTracer* trc, T* thing, const char* name); + +// Trace a root edge that uses the base GC thing type, instead of a more +// specific type. +void +TraceGenericPointerRoot(JSTracer* trc, gc::Cell** thingp, const char* name); + +// Trace a non-root edge that uses the base GC thing type, instead of a more +// specific type. +void +TraceManuallyBarrieredGenericPointerEdge(JSTracer* trc, gc::Cell** thingp, const char* name); + +// Deprecated. Please use one of the strongly typed variants above. +void +TraceChildren(JSTracer* trc, void* thing, JS::TraceKind kind); + +namespace gc { + +// Trace through a shape or group iteratively during cycle collection to avoid +// deep or infinite recursion. +void +TraceCycleCollectorChildren(JS::CallbackTracer* trc, Shape* shape); +void +TraceCycleCollectorChildren(JS::CallbackTracer* trc, ObjectGroup* group); + +} // namespace gc +} // namespace js + +#endif /* js_Tracer_h */ diff --git a/js/src/gc/Verifier.cpp b/js/src/gc/Verifier.cpp new file mode 100644 index 000000000..dd4031606 --- /dev/null +++ b/js/src/gc/Verifier.cpp @@ -0,0 +1,569 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifdef MOZ_VALGRIND +# include <valgrind/memcheck.h> +#endif + +#include "mozilla/IntegerPrintfMacros.h" +#include "mozilla/Sprintf.h" + +#include "jscntxt.h" +#include "jsgc.h" +#include "jsprf.h" + +#include "gc/GCInternals.h" +#include "gc/Zone.h" +#include "js/GCAPI.h" +#include "js/HashTable.h" + +#include "jscntxtinlines.h" +#include "jsgcinlines.h" + +using namespace js; +using namespace js::gc; + +#ifdef JS_GC_ZEAL + +/* + * Write barrier verification + * + * The next few functions are for write barrier verification. + * + * The VerifyBarriers function is a shorthand. It checks if a verification phase + * is currently running. If not, it starts one. Otherwise, it ends the current + * phase and starts a new one. + * + * The user can adjust the frequency of verifications, which causes + * VerifyBarriers to be a no-op all but one out of N calls. However, if the + * |always| parameter is true, it starts a new phase no matter what. + * + * Pre-Barrier Verifier: + * When StartVerifyBarriers is called, a snapshot is taken of all objects in + * the GC heap and saved in an explicit graph data structure. Later, + * EndVerifyBarriers traverses the heap again. Any pointer values that were in + * the snapshot and are no longer found must be marked; otherwise an assertion + * triggers. Note that we must not GC in between starting and finishing a + * verification phase. + */ + +struct EdgeValue +{ + void* thing; + JS::TraceKind kind; + const char* label; +}; + +struct VerifyNode +{ + void* thing; + JS::TraceKind kind; + uint32_t count; + EdgeValue edges[1]; +}; + +typedef HashMap<void*, VerifyNode*, DefaultHasher<void*>, SystemAllocPolicy> NodeMap; + +/* + * The verifier data structures are simple. The entire graph is stored in a + * single block of memory. At the beginning is a VerifyNode for the root + * node. It is followed by a sequence of EdgeValues--the exact number is given + * in the node. After the edges come more nodes and their edges. + * + * The edgeptr and term fields are used to allocate out of the block of memory + * for the graph. If we run out of memory (i.e., if edgeptr goes beyond term), + * we just abandon the verification. + * + * The nodemap field is a hashtable that maps from the address of the GC thing + * to the VerifyNode that represents it. + */ +class js::VerifyPreTracer final : public JS::CallbackTracer +{ + JS::AutoDisableGenerationalGC noggc; + + void onChild(const JS::GCCellPtr& thing) override; + + public: + /* The gcNumber when the verification began. */ + uint64_t number; + + /* This counts up to gcZealFrequency to decide whether to verify. */ + int count; + + /* This graph represents the initial GC "snapshot". */ + VerifyNode* curnode; + VerifyNode* root; + char* edgeptr; + char* term; + NodeMap nodemap; + + explicit VerifyPreTracer(JSRuntime* rt) + : JS::CallbackTracer(rt), noggc(rt), number(rt->gc.gcNumber()), count(0), curnode(nullptr), + root(nullptr), edgeptr(nullptr), term(nullptr) + {} + + ~VerifyPreTracer() { + js_free(root); + } +}; + +/* + * This function builds up the heap snapshot by adding edges to the current + * node. + */ +void +VerifyPreTracer::onChild(const JS::GCCellPtr& thing) +{ + MOZ_ASSERT(!IsInsideNursery(thing.asCell())); + + // Skip things in other runtimes. + if (thing.asCell()->asTenured().runtimeFromAnyThread() != runtime()) + return; + + edgeptr += sizeof(EdgeValue); + if (edgeptr >= term) { + edgeptr = term; + return; + } + + VerifyNode* node = curnode; + uint32_t i = node->count; + + node->edges[i].thing = thing.asCell(); + node->edges[i].kind = thing.kind(); + node->edges[i].label = contextName(); + node->count++; +} + +static VerifyNode* +MakeNode(VerifyPreTracer* trc, void* thing, JS::TraceKind kind) +{ + NodeMap::AddPtr p = trc->nodemap.lookupForAdd(thing); + if (!p) { + VerifyNode* node = (VerifyNode*)trc->edgeptr; + trc->edgeptr += sizeof(VerifyNode) - sizeof(EdgeValue); + if (trc->edgeptr >= trc->term) { + trc->edgeptr = trc->term; + return nullptr; + } + + node->thing = thing; + node->count = 0; + node->kind = kind; + if (!trc->nodemap.add(p, thing, node)) { + trc->edgeptr = trc->term; + return nullptr; + } + + return node; + } + return nullptr; +} + +static VerifyNode* +NextNode(VerifyNode* node) +{ + if (node->count == 0) + return (VerifyNode*)((char*)node + sizeof(VerifyNode) - sizeof(EdgeValue)); + else + return (VerifyNode*)((char*)node + sizeof(VerifyNode) + + sizeof(EdgeValue)*(node->count - 1)); +} + +void +gc::GCRuntime::startVerifyPreBarriers() +{ + if (verifyPreData || isIncrementalGCInProgress()) + return; + + if (IsIncrementalGCUnsafe(rt) != AbortReason::None) + return; + + number++; + + VerifyPreTracer* trc = js_new<VerifyPreTracer>(rt); + if (!trc) + return; + + AutoPrepareForTracing prep(rt->contextFromMainThread(), WithAtoms); + + for (auto chunk = allNonEmptyChunks(); !chunk.done(); chunk.next()) + chunk->bitmap.clear(); + + gcstats::AutoPhase ap(stats, gcstats::PHASE_TRACE_HEAP); + + const size_t size = 64 * 1024 * 1024; + trc->root = (VerifyNode*)js_malloc(size); + if (!trc->root) + goto oom; + trc->edgeptr = (char*)trc->root; + trc->term = trc->edgeptr + size; + + if (!trc->nodemap.init()) + goto oom; + + /* Create the root node. */ + trc->curnode = MakeNode(trc, nullptr, JS::TraceKind(0)); + + incrementalState = State::MarkRoots; + + /* Make all the roots be edges emanating from the root node. */ + traceRuntime(trc, prep.session().lock); + + VerifyNode* node; + node = trc->curnode; + if (trc->edgeptr == trc->term) + goto oom; + + /* For each edge, make a node for it if one doesn't already exist. */ + while ((char*)node < trc->edgeptr) { + for (uint32_t i = 0; i < node->count; i++) { + EdgeValue& e = node->edges[i]; + VerifyNode* child = MakeNode(trc, e.thing, e.kind); + if (child) { + trc->curnode = child; + js::TraceChildren(trc, e.thing, e.kind); + } + if (trc->edgeptr == trc->term) + goto oom; + } + + node = NextNode(node); + } + + verifyPreData = trc; + incrementalState = State::Mark; + marker.start(); + + for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) { + PurgeJITCaches(zone); + if (!zone->usedByExclusiveThread) { + zone->setNeedsIncrementalBarrier(true, Zone::UpdateJit); + zone->arenas.purge(); + } + } + + return; + +oom: + incrementalState = State::NotActive; + js_delete(trc); + verifyPreData = nullptr; +} + +static bool +IsMarkedOrAllocated(TenuredCell* cell) +{ + return cell->isMarked() || cell->arena()->allocatedDuringIncremental; +} + +struct CheckEdgeTracer : public JS::CallbackTracer { + VerifyNode* node; + explicit CheckEdgeTracer(JSRuntime* rt) : JS::CallbackTracer(rt), node(nullptr) {} + void onChild(const JS::GCCellPtr& thing) override; +}; + +static const uint32_t MAX_VERIFIER_EDGES = 1000; + +/* + * This function is called by EndVerifyBarriers for every heap edge. If the edge + * already existed in the original snapshot, we "cancel it out" by overwriting + * it with nullptr. EndVerifyBarriers later asserts that the remaining + * non-nullptr edges (i.e., the ones from the original snapshot that must have + * been modified) must point to marked objects. + */ +void +CheckEdgeTracer::onChild(const JS::GCCellPtr& thing) +{ + // Skip things in other runtimes. + if (thing.asCell()->asTenured().runtimeFromAnyThread() != runtime()) + return; + + /* Avoid n^2 behavior. */ + if (node->count > MAX_VERIFIER_EDGES) + return; + + for (uint32_t i = 0; i < node->count; i++) { + if (node->edges[i].thing == thing.asCell()) { + MOZ_ASSERT(node->edges[i].kind == thing.kind()); + node->edges[i].thing = nullptr; + return; + } + } +} + +void +js::gc::AssertSafeToSkipBarrier(TenuredCell* thing) +{ + Zone* zone = thing->zoneFromAnyThread(); + MOZ_ASSERT(!zone->needsIncrementalBarrier() || zone->isAtomsZone()); +} + +static bool +IsMarkedOrAllocated(const EdgeValue& edge) +{ + if (!edge.thing || IsMarkedOrAllocated(TenuredCell::fromPointer(edge.thing))) + return true; + + // Permanent atoms and well-known symbols aren't marked during graph traversal. + if (edge.kind == JS::TraceKind::String && static_cast<JSString*>(edge.thing)->isPermanentAtom()) + return true; + if (edge.kind == JS::TraceKind::Symbol && static_cast<JS::Symbol*>(edge.thing)->isWellKnownSymbol()) + return true; + + return false; +} + +void +gc::GCRuntime::endVerifyPreBarriers() +{ + VerifyPreTracer* trc = verifyPreData; + + if (!trc) + return; + + MOZ_ASSERT(!JS::IsGenerationalGCEnabled(rt)); + + AutoPrepareForTracing prep(rt->contextFromMainThread(), SkipAtoms); + + bool compartmentCreated = false; + + /* We need to disable barriers before tracing, which may invoke barriers. */ + for (ZonesIter zone(rt, WithAtoms); !zone.done(); zone.next()) { + if (!zone->needsIncrementalBarrier()) + compartmentCreated = true; + + zone->setNeedsIncrementalBarrier(false, Zone::UpdateJit); + PurgeJITCaches(zone); + } + + /* + * We need to bump gcNumber so that the methodjit knows that jitcode has + * been discarded. + */ + MOZ_ASSERT(trc->number == number); + number++; + + verifyPreData = nullptr; + incrementalState = State::NotActive; + + if (!compartmentCreated && IsIncrementalGCUnsafe(rt) == AbortReason::None) { + CheckEdgeTracer cetrc(rt); + + /* Start after the roots. */ + VerifyNode* node = NextNode(trc->root); + while ((char*)node < trc->edgeptr) { + cetrc.node = node; + js::TraceChildren(&cetrc, node->thing, node->kind); + + if (node->count <= MAX_VERIFIER_EDGES) { + for (uint32_t i = 0; i < node->count; i++) { + EdgeValue& edge = node->edges[i]; + if (!IsMarkedOrAllocated(edge)) { + char msgbuf[1024]; + SprintfLiteral(msgbuf, + "[barrier verifier] Unmarked edge: %s %p '%s' edge to %s %p", + JS::GCTraceKindToAscii(node->kind), node->thing, + edge.label, + JS::GCTraceKindToAscii(edge.kind), edge.thing); + MOZ_ReportAssertionFailure(msgbuf, __FILE__, __LINE__); + MOZ_CRASH(); + } + } + } + + node = NextNode(node); + } + } + + marker.reset(); + marker.stop(); + + js_delete(trc); +} + +/*** Barrier Verifier Scheduling ***/ + +void +gc::GCRuntime::verifyPreBarriers() +{ + if (verifyPreData) + endVerifyPreBarriers(); + else + startVerifyPreBarriers(); +} + +void +gc::VerifyBarriers(JSRuntime* rt, VerifierType type) +{ + if (type == PreBarrierVerifier) + rt->gc.verifyPreBarriers(); +} + +void +gc::GCRuntime::maybeVerifyPreBarriers(bool always) +{ + if (!hasZealMode(ZealMode::VerifierPre)) + return; + + if (rt->mainThread.suppressGC) + return; + + if (verifyPreData) { + if (++verifyPreData->count < zealFrequency && !always) + return; + + endVerifyPreBarriers(); + } + + startVerifyPreBarriers(); +} + +void +js::gc::MaybeVerifyBarriers(JSContext* cx, bool always) +{ + GCRuntime* gc = &cx->runtime()->gc; + gc->maybeVerifyPreBarriers(always); +} + +void +js::gc::GCRuntime::finishVerifier() +{ + if (verifyPreData) { + js_delete(verifyPreData); + verifyPreData = nullptr; + } +} + +#endif /* JS_GC_ZEAL */ + +#ifdef JSGC_HASH_TABLE_CHECKS + +class CheckHeapTracer : public JS::CallbackTracer +{ + public: + explicit CheckHeapTracer(JSRuntime* rt); + bool init(); + void check(AutoLockForExclusiveAccess& lock); + + private: + void onChild(const JS::GCCellPtr& thing) override; + + struct WorkItem { + WorkItem(JS::GCCellPtr thing, const char* name, int parentIndex) + : thing(thing), name(name), parentIndex(parentIndex), processed(false) + {} + + JS::GCCellPtr thing; + const char* name; + int parentIndex; + bool processed; + }; + + JSRuntime* rt; + bool oom; + size_t failures; + HashSet<Cell*, DefaultHasher<Cell*>, SystemAllocPolicy> visited; + Vector<WorkItem, 0, SystemAllocPolicy> stack; + int parentIndex; +}; + +CheckHeapTracer::CheckHeapTracer(JSRuntime* rt) + : CallbackTracer(rt, TraceWeakMapKeysValues), + rt(rt), + oom(false), + failures(0), + parentIndex(-1) +{ +#ifdef DEBUG + setCheckEdges(false); +#endif +} + +bool +CheckHeapTracer::init() +{ + return visited.init(); +} + +inline static bool +IsValidGCThingPointer(Cell* cell) +{ + return (uintptr_t(cell) & CellMask) == 0; +} + +void +CheckHeapTracer::onChild(const JS::GCCellPtr& thing) +{ + Cell* cell = thing.asCell(); + if (visited.lookup(cell)) + return; + + if (!visited.put(cell)) { + oom = true; + return; + } + + if (!IsValidGCThingPointer(cell) || !IsGCThingValidAfterMovingGC(cell)) + { + failures++; + fprintf(stderr, "Bad pointer %p\n", cell); + const char* name = contextName(); + for (int index = parentIndex; index != -1; index = stack[index].parentIndex) { + const WorkItem& parent = stack[index]; + cell = parent.thing.asCell(); + fprintf(stderr, " from %s %p %s edge\n", + GCTraceKindToAscii(cell->getTraceKind()), cell, name); + name = parent.name; + } + fprintf(stderr, " from root %s\n", name); + return; + } + + WorkItem item(thing, contextName(), parentIndex); + if (!stack.append(item)) + oom = true; +} + +void +CheckHeapTracer::check(AutoLockForExclusiveAccess& lock) +{ + // The analysis thinks that traceRuntime might GC by calling a GC callback. + JS::AutoSuppressGCAnalysis nogc; + if (!rt->isBeingDestroyed()) + rt->gc.traceRuntime(this, lock); + + while (!stack.empty()) { + WorkItem item = stack.back(); + if (item.processed) { + stack.popBack(); + } else { + parentIndex = stack.length() - 1; + TraceChildren(this, item.thing); + stack.back().processed = true; + } + } + + if (oom) + return; + + if (failures) { + fprintf(stderr, "Heap check: %" PRIuSIZE " failure(s) out of %" PRIu32 " pointers checked\n", + failures, visited.count()); + } + MOZ_RELEASE_ASSERT(failures == 0); +} + +void +js::gc::CheckHeapAfterGC(JSRuntime* rt) +{ + AutoTraceSession session(rt, JS::HeapState::Tracing); + CheckHeapTracer tracer(rt); + if (tracer.init()) + tracer.check(session.lock); +} + +#endif /* JSGC_HASH_TABLE_CHECKS */ diff --git a/js/src/gc/Zone.cpp b/js/src/gc/Zone.cpp new file mode 100644 index 000000000..ed099341c --- /dev/null +++ b/js/src/gc/Zone.cpp @@ -0,0 +1,471 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gc/Zone.h" + +#include "jsgc.h" + +#include "gc/Policy.h" +#include "jit/BaselineJIT.h" +#include "jit/Ion.h" +#include "jit/JitCompartment.h" +#include "vm/Debugger.h" +#include "vm/Runtime.h" + +#include "jscompartmentinlines.h" +#include "jsgcinlines.h" + +using namespace js; +using namespace js::gc; + +Zone * const Zone::NotOnList = reinterpret_cast<Zone*>(1); + +JS::Zone::Zone(JSRuntime* rt) + : JS::shadow::Zone(rt, &rt->gc.marker), + debuggers(nullptr), + suppressAllocationMetadataBuilder(false), + arenas(rt), + types(this), + compartments(), + gcGrayRoots(), + gcWeakKeys(SystemAllocPolicy(), rt->randomHashCodeScrambler()), + typeDescrObjects(this, SystemAllocPolicy()), + gcMallocBytes(0), + gcMallocGCTriggered(false), + usage(&rt->gc.usage), + gcDelayBytes(0), + propertyTree(this), + baseShapes(this, BaseShapeSet()), + initialShapes(this, InitialShapeSet()), + data(nullptr), + isSystem(false), + usedByExclusiveThread(false), + active(false), + jitZone_(nullptr), + gcState_(NoGC), + gcScheduled_(false), + gcPreserveCode_(false), + jitUsingBarriers_(false), + keepShapeTables_(false), + listNext_(NotOnList) +{ + /* Ensure that there are no vtables to mess us up here. */ + MOZ_ASSERT(reinterpret_cast<JS::shadow::Zone*>(this) == + static_cast<JS::shadow::Zone*>(this)); + + AutoLockGC lock(rt); + threshold.updateAfterGC(8192, GC_NORMAL, rt->gc.tunables, rt->gc.schedulingState, lock); + setGCMaxMallocBytes(rt->gc.maxMallocBytesAllocated() * 0.9); +} + +Zone::~Zone() +{ + JSRuntime* rt = runtimeFromMainThread(); + if (this == rt->gc.systemZone) + rt->gc.systemZone = nullptr; + + js_delete(debuggers); + js_delete(jitZone_); + +#ifdef DEBUG + // Avoid assertion destroying the weak map list if the embedding leaked GC things. + if (!rt->gc.shutdownCollectedEverything()) + gcWeakMapList.clear(); +#endif +} + +bool Zone::init(bool isSystemArg) +{ + isSystem = isSystemArg; + return uniqueIds_.init() && + gcZoneGroupEdges.init() && + gcWeakKeys.init() && + typeDescrObjects.init(); +} + +void +Zone::setNeedsIncrementalBarrier(bool needs, ShouldUpdateJit updateJit) +{ + if (updateJit == UpdateJit && needs != jitUsingBarriers_) { + jit::ToggleBarriers(this, needs); + jitUsingBarriers_ = needs; + } + + MOZ_ASSERT_IF(needs && isAtomsZone(), !runtimeFromMainThread()->exclusiveThreadsPresent()); + MOZ_ASSERT_IF(needs, canCollect()); + needsIncrementalBarrier_ = needs; +} + +void +Zone::resetGCMallocBytes() +{ + gcMallocBytes = ptrdiff_t(gcMaxMallocBytes); + gcMallocGCTriggered = false; +} + +void +Zone::setGCMaxMallocBytes(size_t value) +{ + /* + * For compatibility treat any value that exceeds PTRDIFF_T_MAX to + * mean that value. + */ + gcMaxMallocBytes = (ptrdiff_t(value) >= 0) ? value : size_t(-1) >> 1; + resetGCMallocBytes(); +} + +void +Zone::onTooMuchMalloc() +{ + if (!gcMallocGCTriggered) { + GCRuntime& gc = runtimeFromAnyThread()->gc; + gcMallocGCTriggered = gc.triggerZoneGC(this, JS::gcreason::TOO_MUCH_MALLOC); + } +} + +void +Zone::beginSweepTypes(FreeOp* fop, bool releaseTypes) +{ + // Periodically release observed types for all scripts. This is safe to + // do when there are no frames for the zone on the stack. + if (active) + releaseTypes = false; + + AutoClearTypeInferenceStateOnOOM oom(this); + types.beginSweep(fop, releaseTypes, oom); +} + +Zone::DebuggerVector* +Zone::getOrCreateDebuggers(JSContext* cx) +{ + if (debuggers) + return debuggers; + + debuggers = js_new<DebuggerVector>(); + if (!debuggers) + ReportOutOfMemory(cx); + return debuggers; +} + +void +Zone::sweepBreakpoints(FreeOp* fop) +{ + if (fop->runtime()->debuggerList.isEmpty()) + return; + + /* + * Sweep all compartments in a zone at the same time, since there is no way + * to iterate over the scripts belonging to a single compartment in a zone. + */ + + MOZ_ASSERT(isGCSweepingOrCompacting()); + for (auto iter = cellIter<JSScript>(); !iter.done(); iter.next()) { + JSScript* script = iter; + if (!script->hasAnyBreakpointsOrStepMode()) + continue; + + bool scriptGone = IsAboutToBeFinalizedUnbarriered(&script); + MOZ_ASSERT(script == iter); + for (unsigned i = 0; i < script->length(); i++) { + BreakpointSite* site = script->getBreakpointSite(script->offsetToPC(i)); + if (!site) + continue; + + Breakpoint* nextbp; + for (Breakpoint* bp = site->firstBreakpoint(); bp; bp = nextbp) { + nextbp = bp->nextInSite(); + GCPtrNativeObject& dbgobj = bp->debugger->toJSObjectRef(); + + // If we are sweeping, then we expect the script and the + // debugger object to be swept in the same zone group, except if + // the breakpoint was added after we computed the zone + // groups. In this case both script and debugger object must be + // live. + MOZ_ASSERT_IF(isGCSweeping() && dbgobj->zone()->isCollecting(), + dbgobj->zone()->isGCSweeping() || + (!scriptGone && dbgobj->asTenured().isMarked())); + + bool dying = scriptGone || IsAboutToBeFinalized(&dbgobj); + MOZ_ASSERT_IF(!dying, !IsAboutToBeFinalized(&bp->getHandlerRef())); + if (dying) + bp->destroy(fop); + } + } + } +} + +void +Zone::sweepWeakMaps() +{ + /* Finalize unreachable (key,value) pairs in all weak maps. */ + WeakMapBase::sweepZone(this); +} + +void +Zone::discardJitCode(FreeOp* fop, bool discardBaselineCode) +{ + if (!jitZone()) + return; + + if (isPreservingCode()) { + PurgeJITCaches(this); + } else { + + if (discardBaselineCode) { +#ifdef DEBUG + /* Assert no baseline scripts are marked as active. */ + for (auto script = cellIter<JSScript>(); !script.done(); script.next()) + MOZ_ASSERT_IF(script->hasBaselineScript(), !script->baselineScript()->active()); +#endif + + /* Mark baseline scripts on the stack as active. */ + jit::MarkActiveBaselineScripts(this); + } + + /* Only mark OSI points if code is being discarded. */ + jit::InvalidateAll(fop, this); + + for (auto script = cellIter<JSScript>(); !script.done(); script.next()) { + jit::FinishInvalidation(fop, script); + + /* + * Discard baseline script if it's not marked as active. Note that + * this also resets the active flag. + */ + if (discardBaselineCode) + jit::FinishDiscardBaselineScript(fop, script); + + /* + * Warm-up counter for scripts are reset on GC. After discarding code we + * need to let it warm back up to get information such as which + * opcodes are setting array holes or accessing getter properties. + */ + script->resetWarmUpCounter(); + } + + /* + * When scripts contains pointers to nursery things, the store buffer + * can contain entries that point into the optimized stub space. Since + * this method can be called outside the context of a GC, this situation + * could result in us trying to mark invalid store buffer entries. + * + * Defer freeing any allocated blocks until after the next minor GC. + */ + if (discardBaselineCode) + jitZone()->optimizedStubSpace()->freeAllAfterMinorGC(fop->runtime()); + } +} + +#ifdef JSGC_HASH_TABLE_CHECKS +void +JS::Zone::checkUniqueIdTableAfterMovingGC() +{ + for (UniqueIdMap::Enum e(uniqueIds_); !e.empty(); e.popFront()) + js::gc::CheckGCThingAfterMovingGC(e.front().key()); +} +#endif + +uint64_t +Zone::gcNumber() +{ + // Zones in use by exclusive threads are not collected, and threads using + // them cannot access the main runtime's gcNumber without racing. + return usedByExclusiveThread ? 0 : runtimeFromMainThread()->gc.gcNumber(); +} + +js::jit::JitZone* +Zone::createJitZone(JSContext* cx) +{ + MOZ_ASSERT(!jitZone_); + + if (!cx->runtime()->getJitRuntime(cx)) + return nullptr; + + jitZone_ = cx->new_<js::jit::JitZone>(); + return jitZone_; +} + +bool +Zone::hasMarkedCompartments() +{ + for (CompartmentsInZoneIter comp(this); !comp.done(); comp.next()) { + if (comp->marked) + return true; + } + return false; +} + +bool +Zone::canCollect() +{ + // Zones cannot be collected while in use by other threads. + if (usedByExclusiveThread) + return false; + JSRuntime* rt = runtimeFromAnyThread(); + if (isAtomsZone() && rt->exclusiveThreadsPresent()) + return false; + return true; +} + +void +Zone::notifyObservingDebuggers() +{ + for (CompartmentsInZoneIter comps(this); !comps.done(); comps.next()) { + JSRuntime* rt = runtimeFromAnyThread(); + RootedGlobalObject global(rt->contextFromMainThread(), comps->unsafeUnbarrieredMaybeGlobal()); + if (!global) + continue; + + GlobalObject::DebuggerVector* dbgs = global->getDebuggers(); + if (!dbgs) + continue; + + for (GlobalObject::DebuggerVector::Range r = dbgs->all(); !r.empty(); r.popFront()) { + if (!r.front()->debuggeeIsBeingCollected(rt->gc.majorGCCount())) { +#ifdef DEBUG + fprintf(stderr, + "OOM while notifying observing Debuggers of a GC: The onGarbageCollection\n" + "hook will not be fired for this GC for some Debuggers!\n"); +#endif + return; + } + } + } +} + +bool +js::ZonesIter::atAtomsZone(JSRuntime* rt) +{ + return rt->isAtomsZone(*it); +} + +bool +Zone::isOnList() const +{ + return listNext_ != NotOnList; +} + +Zone* +Zone::nextZone() const +{ + MOZ_ASSERT(isOnList()); + return listNext_; +} + +void +Zone::clearTables() +{ + if (baseShapes.initialized()) + baseShapes.clear(); + if (initialShapes.initialized()) + initialShapes.clear(); +} + +void +Zone::fixupAfterMovingGC() +{ + fixupInitialShapeTable(); +} + +ZoneList::ZoneList() + : head(nullptr), tail(nullptr) +{} + +ZoneList::ZoneList(Zone* zone) + : head(zone), tail(zone) +{ + MOZ_RELEASE_ASSERT(!zone->isOnList()); + zone->listNext_ = nullptr; +} + +ZoneList::~ZoneList() +{ + MOZ_ASSERT(isEmpty()); +} + +void +ZoneList::check() const +{ +#ifdef DEBUG + MOZ_ASSERT((head == nullptr) == (tail == nullptr)); + if (!head) + return; + + Zone* zone = head; + for (;;) { + MOZ_ASSERT(zone && zone->isOnList()); + if (zone == tail) + break; + zone = zone->listNext_; + } + MOZ_ASSERT(!zone->listNext_); +#endif +} + +bool +ZoneList::isEmpty() const +{ + return head == nullptr; +} + +Zone* +ZoneList::front() const +{ + MOZ_ASSERT(!isEmpty()); + MOZ_ASSERT(head->isOnList()); + return head; +} + +void +ZoneList::append(Zone* zone) +{ + ZoneList singleZone(zone); + transferFrom(singleZone); +} + +void +ZoneList::transferFrom(ZoneList& other) +{ + check(); + other.check(); + MOZ_ASSERT(tail != other.tail); + + if (tail) + tail->listNext_ = other.head; + else + head = other.head; + tail = other.tail; + + other.head = nullptr; + other.tail = nullptr; +} + +void +ZoneList::removeFront() +{ + MOZ_ASSERT(!isEmpty()); + check(); + + Zone* front = head; + head = head->listNext_; + if (!head) + tail = nullptr; + + front->listNext_ = Zone::NotOnList; +} + +void +ZoneList::clear() +{ + while (!isEmpty()) + removeFront(); +} + +JS_PUBLIC_API(void) +JS::shadow::RegisterWeakCache(JS::Zone* zone, WeakCache<void*>* cachep) +{ + zone->registerWeakCache(cachep); +} diff --git a/js/src/gc/Zone.h b/js/src/gc/Zone.h new file mode 100644 index 000000000..a3a6dc07f --- /dev/null +++ b/js/src/gc/Zone.h @@ -0,0 +1,743 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * vim: set ts=8 sts=4 et sw=4 tw=99: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef gc_Zone_h +#define gc_Zone_h + +#include "mozilla/Atomics.h" +#include "mozilla/MemoryReporting.h" + +#include "jscntxt.h" + +#include "ds/SplayTree.h" +#include "gc/FindSCCs.h" +#include "gc/GCRuntime.h" +#include "js/GCHashTable.h" +#include "js/TracingAPI.h" +#include "vm/MallocProvider.h" +#include "vm/TypeInference.h" + +namespace js { + +namespace jit { +class JitZone; +} // namespace jit + +namespace gc { + +// This class encapsulates the data that determines when we need to do a zone GC. +class ZoneHeapThreshold +{ + // The "growth factor" for computing our next thresholds after a GC. + double gcHeapGrowthFactor_; + + // GC trigger threshold for allocations on the GC heap. + mozilla::Atomic<size_t, mozilla::Relaxed> gcTriggerBytes_; + + public: + ZoneHeapThreshold() + : gcHeapGrowthFactor_(3.0), + gcTriggerBytes_(0) + {} + + double gcHeapGrowthFactor() const { return gcHeapGrowthFactor_; } + size_t gcTriggerBytes() const { return gcTriggerBytes_; } + double allocTrigger(bool highFrequencyGC) const; + + void updateAfterGC(size_t lastBytes, JSGCInvocationKind gckind, + const GCSchedulingTunables& tunables, const GCSchedulingState& state, + const AutoLockGC& lock); + void updateForRemovedArena(const GCSchedulingTunables& tunables); + + private: + static double computeZoneHeapGrowthFactorForHeapSize(size_t lastBytes, + const GCSchedulingTunables& tunables, + const GCSchedulingState& state); + static size_t computeZoneTriggerBytes(double growthFactor, size_t lastBytes, + JSGCInvocationKind gckind, + const GCSchedulingTunables& tunables, + const AutoLockGC& lock); +}; + +struct ZoneComponentFinder : public ComponentFinder<JS::Zone, ZoneComponentFinder> +{ + ZoneComponentFinder(uintptr_t sl, AutoLockForExclusiveAccess& lock) + : ComponentFinder<JS::Zone, ZoneComponentFinder>(sl), lock(lock) + {} + + AutoLockForExclusiveAccess& lock; +}; + +struct UniqueIdGCPolicy { + static bool needsSweep(Cell** cell, uint64_t* value); +}; + +// Maps a Cell* to a unique, 64bit id. +using UniqueIdMap = GCHashMap<Cell*, + uint64_t, + PointerHasher<Cell*, 3>, + SystemAllocPolicy, + UniqueIdGCPolicy>; + +extern uint64_t NextCellUniqueId(JSRuntime* rt); + +template <typename T> +class ZoneCellIter; + +} // namespace gc +} // namespace js + +namespace JS { + +// A zone is a collection of compartments. Every compartment belongs to exactly +// one zone. In Firefox, there is roughly one zone per tab along with a system +// zone for everything else. Zones mainly serve as boundaries for garbage +// collection. Unlike compartments, they have no special security properties. +// +// Every GC thing belongs to exactly one zone. GC things from the same zone but +// different compartments can share an arena (4k page). GC things from different +// zones cannot be stored in the same arena. The garbage collector is capable of +// collecting one zone at a time; it cannot collect at the granularity of +// compartments. +// +// GC things are tied to zones and compartments as follows: +// +// - JSObjects belong to a compartment and cannot be shared between +// compartments. If an object needs to point to a JSObject in a different +// compartment, regardless of zone, it must go through a cross-compartment +// wrapper. Each compartment keeps track of its outgoing wrappers in a table. +// JSObjects find their compartment via their ObjectGroup. +// +// - JSStrings do not belong to any particular compartment, but they do belong +// to a zone. Thus, two different compartments in the same zone can point to a +// JSString. When a string needs to be wrapped, we copy it if it's in a +// different zone and do nothing if it's in the same zone. Thus, transferring +// strings within a zone is very efficient. +// +// - Shapes and base shapes belong to a zone and are shared between compartments +// in that zone where possible. Accessor shapes store getter and setter +// JSObjects which belong to a single compartment, so these shapes and all +// their descendants can't be shared with other compartments. +// +// - Scripts are also compartment-local and cannot be shared. A script points to +// its compartment. +// +// - ObjectGroup and JitCode objects belong to a compartment and cannot be +// shared. There is no mechanism to obtain the compartment from a JitCode +// object. +// +// A zone remains alive as long as any GC things in the zone are alive. A +// compartment remains alive as long as any JSObjects, scripts, shapes, or base +// shapes within it are alive. +// +// We always guarantee that a zone has at least one live compartment by refusing +// to delete the last compartment in a live zone. +struct Zone : public JS::shadow::Zone, + public js::gc::GraphNodeBase<JS::Zone>, + public js::MallocProvider<JS::Zone> +{ + explicit Zone(JSRuntime* rt); + ~Zone(); + MOZ_MUST_USE bool init(bool isSystem); + + void findOutgoingEdges(js::gc::ZoneComponentFinder& finder); + + void discardJitCode(js::FreeOp* fop, bool discardBaselineCode = true); + + void addSizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf, + size_t* typePool, + size_t* baselineStubsOptimized, + size_t* uniqueIdMap, + size_t* shapeTables); + + void resetGCMallocBytes(); + void setGCMaxMallocBytes(size_t value); + void updateMallocCounter(size_t nbytes) { + // Note: this code may be run from worker threads. We tolerate any + // thread races when updating gcMallocBytes. + gcMallocBytes -= ptrdiff_t(nbytes); + if (MOZ_UNLIKELY(isTooMuchMalloc())) + onTooMuchMalloc(); + } + + // Iterate over all cells in the zone. See the definition of ZoneCellIter + // in jsgcinlines.h for the possible arguments and documentation. + template <typename T, typename... Args> + js::gc::ZoneCellIter<T> cellIter(Args&&... args) { + return js::gc::ZoneCellIter<T>(const_cast<Zone*>(this), mozilla::Forward<Args>(args)...); + } + + bool isTooMuchMalloc() const { return gcMallocBytes <= 0; } + void onTooMuchMalloc(); + + MOZ_MUST_USE void* onOutOfMemory(js::AllocFunction allocFunc, size_t nbytes, + void* reallocPtr = nullptr) { + if (!js::CurrentThreadCanAccessRuntime(runtime_)) + return nullptr; + return runtimeFromMainThread()->onOutOfMemory(allocFunc, nbytes, reallocPtr); + } + void reportAllocationOverflow() { js::ReportAllocationOverflow(nullptr); } + + void beginSweepTypes(js::FreeOp* fop, bool releaseTypes); + + bool hasMarkedCompartments(); + + void scheduleGC() { MOZ_ASSERT(!runtimeFromMainThread()->isHeapBusy()); gcScheduled_ = true; } + void unscheduleGC() { gcScheduled_ = false; } + bool isGCScheduled() { return gcScheduled_ && canCollect(); } + + void setPreservingCode(bool preserving) { gcPreserveCode_ = preserving; } + bool isPreservingCode() const { return gcPreserveCode_; } + + bool canCollect(); + + void notifyObservingDebuggers(); + + enum GCState { + NoGC, + Mark, + MarkGray, + Sweep, + Finished, + Compact + }; + void setGCState(GCState state) { + MOZ_ASSERT(runtimeFromMainThread()->isHeapBusy()); + MOZ_ASSERT_IF(state != NoGC, canCollect()); + gcState_ = state; + if (state == Finished) + notifyObservingDebuggers(); + } + + bool isCollecting() const { + if (runtimeFromMainThread()->isHeapCollecting()) + return gcState_ != NoGC; + else + return needsIncrementalBarrier(); + } + + bool isCollectingFromAnyThread() const { + if (runtimeFromAnyThread()->isHeapCollecting()) + return gcState_ != NoGC; + else + return needsIncrementalBarrier(); + } + + // If this returns true, all object tracing must be done with a GC marking + // tracer. + bool requireGCTracer() const { + JSRuntime* rt = runtimeFromAnyThread(); + return rt->isHeapMajorCollecting() && !rt->gc.isHeapCompacting() && gcState_ != NoGC; + } + + bool isGCMarking() { + if (runtimeFromMainThread()->isHeapCollecting()) + return gcState_ == Mark || gcState_ == MarkGray; + else + return needsIncrementalBarrier(); + } + + GCState gcState() const { return gcState_; } + bool wasGCStarted() const { return gcState_ != NoGC; } + bool isGCMarkingBlack() { return gcState_ == Mark; } + bool isGCMarkingGray() { return gcState_ == MarkGray; } + bool isGCSweeping() { return gcState_ == Sweep; } + bool isGCFinished() { return gcState_ == Finished; } + bool isGCCompacting() { return gcState_ == Compact; } + bool isGCSweepingOrCompacting() { return gcState_ == Sweep || gcState_ == Compact; } + + // Get a number that is incremented whenever this zone is collected, and + // possibly at other times too. + uint64_t gcNumber(); + + bool compileBarriers() const { return compileBarriers(needsIncrementalBarrier()); } + bool compileBarriers(bool needsIncrementalBarrier) const { + return needsIncrementalBarrier || + runtimeFromMainThread()->hasZealMode(js::gc::ZealMode::VerifierPre); + } + + enum ShouldUpdateJit { DontUpdateJit, UpdateJit }; + void setNeedsIncrementalBarrier(bool needs, ShouldUpdateJit updateJit); + const bool* addressOfNeedsIncrementalBarrier() const { return &needsIncrementalBarrier_; } + + js::jit::JitZone* getJitZone(JSContext* cx) { return jitZone_ ? jitZone_ : createJitZone(cx); } + js::jit::JitZone* jitZone() { return jitZone_; } + + bool isAtomsZone() const { return runtimeFromAnyThread()->isAtomsZone(this); } + bool isSelfHostingZone() const { return runtimeFromAnyThread()->isSelfHostingZone(this); } + + void prepareForCompacting(); + +#ifdef DEBUG + // For testing purposes, return the index of the zone group which this zone + // was swept in in the last GC. + unsigned lastZoneGroupIndex() { return gcLastZoneGroupIndex; } +#endif + + using DebuggerVector = js::Vector<js::Debugger*, 0, js::SystemAllocPolicy>; + + private: + DebuggerVector* debuggers; + + void sweepBreakpoints(js::FreeOp* fop); + void sweepUniqueIds(js::FreeOp* fop); + void sweepWeakMaps(); + void sweepCompartments(js::FreeOp* fop, bool keepAtleastOne, bool lastGC); + + js::jit::JitZone* createJitZone(JSContext* cx); + + bool isQueuedForBackgroundSweep() { + return isOnList(); + } + + // Side map for storing a unique ids for cells, independent of address. + js::gc::UniqueIdMap uniqueIds_; + + public: + bool hasDebuggers() const { return debuggers && debuggers->length(); } + DebuggerVector* getDebuggers() const { return debuggers; } + DebuggerVector* getOrCreateDebuggers(JSContext* cx); + + void clearTables(); + + /* + * When true, skip calling the metadata callback. We use this: + * - to avoid invoking the callback recursively; + * - to avoid observing lazy prototype setup (which confuses callbacks that + * want to use the types being set up!); + * - to avoid attaching allocation stacks to allocation stack nodes, which + * is silly + * And so on. + */ + bool suppressAllocationMetadataBuilder; + + js::gc::ArenaLists arenas; + + js::TypeZone types; + + /* Live weakmaps in this zone. */ + mozilla::LinkedList<js::WeakMapBase> gcWeakMapList; + + // The set of compartments in this zone. + typedef js::Vector<JSCompartment*, 1, js::SystemAllocPolicy> CompartmentVector; + CompartmentVector compartments; + + // This zone's gray roots. + typedef js::Vector<js::gc::Cell*, 0, js::SystemAllocPolicy> GrayRootVector; + GrayRootVector gcGrayRoots; + + // This zone's weak edges found via graph traversal during marking, + // preserved for re-scanning during sweeping. + using WeakEdges = js::Vector<js::gc::TenuredCell**, 0, js::SystemAllocPolicy>; + WeakEdges gcWeakRefs; + + // List of non-ephemeron weak containers to sweep during beginSweepingZoneGroup. + mozilla::LinkedList<WeakCache<void*>> weakCaches_; + void registerWeakCache(WeakCache<void*>* cachep) { + weakCaches_.insertBack(cachep); + } + + /* + * Mapping from not yet marked keys to a vector of all values that the key + * maps to in any live weak map. + */ + js::gc::WeakKeyTable gcWeakKeys; + + // A set of edges from this zone to other zones. + // + // This is used during GC while calculating zone groups to record edges that + // can't be determined by examining this zone by itself. + ZoneSet gcZoneGroupEdges; + + // Keep track of all TypeDescr and related objects in this compartment. + // This is used by the GC to trace them all first when compacting, since the + // TypedObject trace hook may access these objects. + using TypeDescrObjectSet = js::GCHashSet<js::HeapPtr<JSObject*>, + js::MovableCellHasher<js::HeapPtr<JSObject*>>, + js::SystemAllocPolicy>; + JS::WeakCache<TypeDescrObjectSet> typeDescrObjects; + + + // Malloc counter to measure memory pressure for GC scheduling. It runs from + // gcMaxMallocBytes down to zero. This counter should be used only when it's + // not possible to know the size of a free. + mozilla::Atomic<ptrdiff_t, mozilla::ReleaseAcquire> gcMallocBytes; + + // GC trigger threshold for allocations on the C heap. + size_t gcMaxMallocBytes; + + // Whether a GC has been triggered as a result of gcMallocBytes falling + // below zero. + // + // This should be a bool, but Atomic only supports 32-bit and pointer-sized + // types. + mozilla::Atomic<uint32_t, mozilla::ReleaseAcquire> gcMallocGCTriggered; + + // Track heap usage under this Zone. + js::gc::HeapUsage usage; + + // Thresholds used to trigger GC. + js::gc::ZoneHeapThreshold threshold; + + // Amount of data to allocate before triggering a new incremental slice for + // the current GC. + size_t gcDelayBytes; + + // Shared Shape property tree. + js::PropertyTree propertyTree; + + // Set of all unowned base shapes in the Zone. + JS::WeakCache<js::BaseShapeSet> baseShapes; + + // Set of initial shapes in the Zone. For certain prototypes -- namely, + // those of various builtin classes -- there are two entries: one for a + // lookup via TaggedProto, and one for a lookup via JSProtoKey. See + // InitialShapeProto. + JS::WeakCache<js::InitialShapeSet> initialShapes; + +#ifdef JSGC_HASH_TABLE_CHECKS + void checkInitialShapesTableAfterMovingGC(); + void checkBaseShapeTableAfterMovingGC(); +#endif + void fixupInitialShapeTable(); + void fixupAfterMovingGC(); + + // Per-zone data for use by an embedder. + void* data; + + bool isSystem; + + mozilla::Atomic<bool> usedByExclusiveThread; + + // True when there are active frames. + bool active; + +#ifdef DEBUG + unsigned gcLastZoneGroupIndex; +#endif + + static js::HashNumber UniqueIdToHash(uint64_t uid) { + return js::HashNumber(uid >> 32) ^ js::HashNumber(uid & 0xFFFFFFFF); + } + + // Creates a HashNumber based on getUniqueId. Returns false on OOM. + MOZ_MUST_USE bool getHashCode(js::gc::Cell* cell, js::HashNumber* hashp) { + uint64_t uid; + if (!getUniqueId(cell, &uid)) + return false; + *hashp = UniqueIdToHash(uid); + return true; + } + + // Puts an existing UID in |uidp|, or creates a new UID for this Cell and + // puts that into |uidp|. Returns false on OOM. + MOZ_MUST_USE bool getUniqueId(js::gc::Cell* cell, uint64_t* uidp) { + MOZ_ASSERT(uidp); + MOZ_ASSERT(js::CurrentThreadCanAccessZone(this)); + + // Get an existing uid, if one has been set. + auto p = uniqueIds_.lookupForAdd(cell); + if (p) { + *uidp = p->value(); + return true; + } + + // Set a new uid on the cell. + *uidp = js::gc::NextCellUniqueId(runtimeFromAnyThread()); + if (!uniqueIds_.add(p, cell, *uidp)) + return false; + + // If the cell was in the nursery, hopefully unlikely, then we need to + // tell the nursery about it so that it can sweep the uid if the thing + // does not get tenured. + if (!runtimeFromAnyThread()->gc.nursery.addedUniqueIdToCell(cell)) { + uniqueIds_.remove(cell); + return false; + } + + return true; + } + + js::HashNumber getHashCodeInfallible(js::gc::Cell* cell) { + return UniqueIdToHash(getUniqueIdInfallible(cell)); + } + + uint64_t getUniqueIdInfallible(js::gc::Cell* cell) { + uint64_t uid; + js::AutoEnterOOMUnsafeRegion oomUnsafe; + if (!getUniqueId(cell, &uid)) + oomUnsafe.crash("failed to allocate uid"); + return uid; + } + + // Return true if this cell has a UID associated with it. + MOZ_MUST_USE bool hasUniqueId(js::gc::Cell* cell) { + MOZ_ASSERT(js::CurrentThreadCanAccessZone(this)); + return uniqueIds_.has(cell); + } + + // Transfer an id from another cell. This must only be called on behalf of a + // moving GC. This method is infallible. + void transferUniqueId(js::gc::Cell* tgt, js::gc::Cell* src) { + MOZ_ASSERT(src != tgt); + MOZ_ASSERT(!IsInsideNursery(tgt)); + MOZ_ASSERT(CurrentThreadCanAccessRuntime(runtimeFromMainThread())); + MOZ_ASSERT(js::CurrentThreadCanAccessZone(this)); + uniqueIds_.rekeyIfMoved(src, tgt); + } + + // Remove any unique id associated with this Cell. + void removeUniqueId(js::gc::Cell* cell) { + MOZ_ASSERT(js::CurrentThreadCanAccessZone(this)); + uniqueIds_.remove(cell); + } + + // When finished parsing off-thread, transfer any UIDs we created in the + // off-thread zone into the target zone. + void adoptUniqueIds(JS::Zone* source) { + js::AutoEnterOOMUnsafeRegion oomUnsafe; + for (js::gc::UniqueIdMap::Enum e(source->uniqueIds_); !e.empty(); e.popFront()) { + MOZ_ASSERT(!uniqueIds_.has(e.front().key())); + if (!uniqueIds_.put(e.front().key(), e.front().value())) + oomUnsafe.crash("failed to transfer unique ids from off-main-thread"); + } + source->uniqueIds_.clear(); + } + + JSContext* contextFromMainThread() { + return runtime_->contextFromMainThread(); + } + +#ifdef JSGC_HASH_TABLE_CHECKS + // Assert that the UniqueId table has been redirected successfully. + void checkUniqueIdTableAfterMovingGC(); +#endif + + bool keepShapeTables() const { + return keepShapeTables_; + } + void setKeepShapeTables(bool b) { + keepShapeTables_ = b; + } + + private: + js::jit::JitZone* jitZone_; + + GCState gcState_; + bool gcScheduled_; + bool gcPreserveCode_; + bool jitUsingBarriers_; + bool keepShapeTables_; + + // Allow zones to be linked into a list + friend class js::gc::ZoneList; + static Zone * const NotOnList; + Zone* listNext_; + bool isOnList() const; + Zone* nextZone() const; + + friend bool js::CurrentThreadCanAccessZone(Zone* zone); + friend class js::gc::GCRuntime; +}; + +} // namespace JS + +namespace js { + +// Using the atoms zone without holding the exclusive access lock is dangerous +// because worker threads may be using it simultaneously. Therefore, it's +// better to skip the atoms zone when iterating over zones. If you need to +// iterate over the atoms zone, consider taking the exclusive access lock first. +enum ZoneSelector { + WithAtoms, + SkipAtoms +}; + +class ZonesIter +{ + gc::AutoEnterIteration iterMarker; + JS::Zone** it; + JS::Zone** end; + + public: + ZonesIter(JSRuntime* rt, ZoneSelector selector) : iterMarker(&rt->gc) { + it = rt->gc.zones.begin(); + end = rt->gc.zones.end(); + + if (selector == SkipAtoms) { + MOZ_ASSERT(atAtomsZone(rt)); + it++; + } + } + + bool atAtomsZone(JSRuntime* rt); + + bool done() const { return it == end; } + + void next() { + MOZ_ASSERT(!done()); + do { + it++; + } while (!done() && (*it)->usedByExclusiveThread); + } + + JS::Zone* get() const { + MOZ_ASSERT(!done()); + return *it; + } + + operator JS::Zone*() const { return get(); } + JS::Zone* operator->() const { return get(); } +}; + +struct CompartmentsInZoneIter +{ + explicit CompartmentsInZoneIter(JS::Zone* zone) : zone(zone) { + it = zone->compartments.begin(); + } + + bool done() const { + MOZ_ASSERT(it); + return it < zone->compartments.begin() || + it >= zone->compartments.end(); + } + void next() { + MOZ_ASSERT(!done()); + it++; + } + + JSCompartment* get() const { + MOZ_ASSERT(it); + return *it; + } + + operator JSCompartment*() const { return get(); } + JSCompartment* operator->() const { return get(); } + + private: + JS::Zone* zone; + JSCompartment** it; + + CompartmentsInZoneIter() + : zone(nullptr), it(nullptr) + {} + + // This is for the benefit of CompartmentsIterT::comp. + friend class mozilla::Maybe<CompartmentsInZoneIter>; +}; + +// This iterator iterates over all the compartments in a given set of zones. The +// set of zones is determined by iterating ZoneIterT. +template<class ZonesIterT> +class CompartmentsIterT +{ + gc::AutoEnterIteration iterMarker; + ZonesIterT zone; + mozilla::Maybe<CompartmentsInZoneIter> comp; + + public: + explicit CompartmentsIterT(JSRuntime* rt) + : iterMarker(&rt->gc), zone(rt) + { + if (zone.done()) + comp.emplace(); + else + comp.emplace(zone); + } + + CompartmentsIterT(JSRuntime* rt, ZoneSelector selector) + : iterMarker(&rt->gc), zone(rt, selector) + { + if (zone.done()) + comp.emplace(); + else + comp.emplace(zone); + } + + bool done() const { return zone.done(); } + + void next() { + MOZ_ASSERT(!done()); + MOZ_ASSERT(!comp.ref().done()); + comp->next(); + if (comp->done()) { + comp.reset(); + zone.next(); + if (!zone.done()) + comp.emplace(zone); + } + } + + JSCompartment* get() const { + MOZ_ASSERT(!done()); + return *comp; + } + + operator JSCompartment*() const { return get(); } + JSCompartment* operator->() const { return get(); } +}; + +typedef CompartmentsIterT<ZonesIter> CompartmentsIter; + +/* + * Allocation policy that uses Zone::pod_malloc and friends, so that memory + * pressure is accounted for on the zone. This is suitable for memory associated + * with GC things allocated in the zone. + * + * Since it doesn't hold a JSContext (those may not live long enough), it can't + * report out-of-memory conditions itself; the caller must check for OOM and + * take the appropriate action. + * + * FIXME bug 647103 - replace these *AllocPolicy names. + */ +class ZoneAllocPolicy +{ + Zone* const zone; + + public: + MOZ_IMPLICIT ZoneAllocPolicy(Zone* zone) : zone(zone) {} + + template <typename T> + T* maybe_pod_malloc(size_t numElems) { + return zone->maybe_pod_malloc<T>(numElems); + } + + template <typename T> + T* maybe_pod_calloc(size_t numElems) { + return zone->maybe_pod_calloc<T>(numElems); + } + + template <typename T> + T* maybe_pod_realloc(T* p, size_t oldSize, size_t newSize) { + return zone->maybe_pod_realloc<T>(p, oldSize, newSize); + } + + template <typename T> + T* pod_malloc(size_t numElems) { + return zone->pod_malloc<T>(numElems); + } + + template <typename T> + T* pod_calloc(size_t numElems) { + return zone->pod_calloc<T>(numElems); + } + + template <typename T> + T* pod_realloc(T* p, size_t oldSize, size_t newSize) { + return zone->pod_realloc<T>(p, oldSize, newSize); + } + + void free_(void* p) { js_free(p); } + void reportAllocOverflow() const {} + + MOZ_MUST_USE bool checkSimulatedOOM() const { + return !js::oom::ShouldFailWithOOM(); + } +}; + +} // namespace js + +#endif // gc_Zone_h |