summaryrefslogtreecommitdiff
path: root/js/src/gc/Nursery.h
blob: 9195e024dcf6781fbfbe6461c1ea7558f80bfa85 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
 * vim: set ts=8 sw=4 et tw=78:
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
 * You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifndef gc_Nursery_h
#define gc_Nursery_h

#include "mozilla/EnumeratedArray.h"

#include "jsalloc.h"
#include "jspubtd.h"

#include "ds/BitArray.h"
#include "gc/Heap.h"
#include "gc/Memory.h"
#include "js/Class.h"
#include "js/GCAPI.h"
#include "js/HashTable.h"
#include "js/HeapAPI.h"
#include "js/Value.h"
#include "js/Vector.h"
#include "vm/SharedMem.h"

#define FOR_EACH_NURSERY_PROFILE_TIME(_)                                      \
   /* Key                       Header text */                                \
    _(Total,                    "total")                                      \
    _(CancelIonCompilations,    "canIon")                                     \
    _(TraceValues,              "mkVals")                                     \
    _(TraceCells,               "mkClls")                                     \
    _(TraceSlots,               "mkSlts")                                     \
    _(TraceWholeCells,          "mcWCll")                                     \
    _(TraceGenericEntries,      "mkGnrc")                                     \
    _(CheckHashTables,          "ckTbls")                                     \
    _(MarkRuntime,              "mkRntm")                                     \
    _(MarkDebugger,             "mkDbgr")                                     \
    _(ClearNewObjectCache,      "clrNOC")                                     \
    _(CollectToFP,              "collct")                                     \
    _(ObjectsTenuredCallback,   "tenCB")                                      \
    _(SweepArrayBufferViewList, "swpABO")                                     \
    _(UpdateJitActivations,     "updtIn")                                     \
    _(FreeMallocedBuffers,      "frSlts")                                     \
    _(ClearStoreBuffer,         "clrSB")                                      \
    _(Sweep,                    "sweep")                                      \
    _(Resize,                   "resize")                                     \
    _(Pretenure,                "pretnr")

namespace JS {
struct Zone;
} // namespace JS

namespace js {

class ObjectElements;
class NativeObject;
class Nursery;
class HeapSlot;

void SetGCZeal(JSRuntime*, uint8_t, uint32_t);

namespace gc {
class AutoMaybeStartBackgroundAllocation;
struct Cell;
class MinorCollectionTracer;
class RelocationOverlay;
struct TenureCountCache;
} /* namespace gc */

namespace jit {
class MacroAssembler;
} // namespace jit

class TenuringTracer : public JSTracer
{
    friend class Nursery;
    Nursery& nursery_;

    // Amount of data moved to the tenured generation during collection.
    size_t tenuredSize;

    // This list is threaded through the Nursery using the space from already
    // moved things. The list is used to fix up the moved things and to find
    // things held live by intra-Nursery pointers.
    gc::RelocationOverlay* head;
    gc::RelocationOverlay** tail;

    TenuringTracer(JSRuntime* rt, Nursery* nursery);

  public:
    const Nursery& nursery() const { return nursery_; }

    // Returns true if the pointer was updated.
    template <typename T> void traverse(T** thingp);
    template <typename T> void traverse(T* thingp);

    void insertIntoFixupList(gc::RelocationOverlay* entry);

    // The store buffers need to be able to call these directly.
    void traceObject(JSObject* src);
    void traceObjectSlots(NativeObject* nobj, uint32_t start, uint32_t length);
    void traceSlots(JS::Value* vp, uint32_t nslots) { traceSlots(vp, vp + nslots); }

  private:
    Nursery& nursery() { return nursery_; }

    JSObject* moveToTenured(JSObject* src);
    size_t moveObjectToTenured(JSObject* dst, JSObject* src, gc::AllocKind dstKind);
    size_t moveElementsToTenured(NativeObject* dst, NativeObject* src, gc::AllocKind dstKind);
    size_t moveSlotsToTenured(NativeObject* dst, NativeObject* src, gc::AllocKind dstKind);

    void traceSlots(JS::Value* vp, JS::Value* end);
};

/*
 * Classes with JSCLASS_SKIP_NURSERY_FINALIZE or Wrapper classes with
 * CROSS_COMPARTMENT flags will not have their finalizer called if they are
 * nursery allocated and not promoted to the tenured heap. The finalizers for
 * these classes must do nothing except free data which was allocated via
 * Nursery::allocateBuffer.
 */
inline bool
CanNurseryAllocateFinalizedClass(const js::Class* const clasp)
{
    MOZ_ASSERT(clasp->hasFinalize());
    return clasp->flags & JSCLASS_SKIP_NURSERY_FINALIZE;
}

class Nursery
{
  public:
    static const size_t Alignment = gc::ChunkSize;
    static const size_t ChunkShift = gc::ChunkShift;

    explicit Nursery(JSRuntime* rt);
    ~Nursery();

    MOZ_MUST_USE bool init(uint32_t maxNurseryBytes, AutoLockGC& lock);

    unsigned maxChunks() const { return maxNurseryChunks_; }
    unsigned numChunks() const { return chunks_.length(); }

    bool exists() const { return maxChunks() != 0; }
    size_t nurserySize() const { return maxChunks() << ChunkShift; }

    void enable();
    void disable();
    bool isEnabled() const { return numChunks() != 0; }

    /* Return true if no allocations have been made since the last collection. */
    bool isEmpty() const;

    /*
     * Check whether an arbitrary pointer is within the nursery. This is
     * slower than IsInsideNursery(Cell*), but works on all types of pointers.
     */
    MOZ_ALWAYS_INLINE bool isInside(gc::Cell* cellp) const = delete;
    MOZ_ALWAYS_INLINE bool isInside(const void* p) const {
        for (auto chunk : chunks_) {
            if (uintptr_t(p) - chunk->start() < gc::ChunkSize)
                return true;
        }
        return false;
    }
    template<typename T>
    bool isInside(const SharedMem<T>& p) const {
        return isInside(p.unwrap(/*safe - used for value in comparison above*/));
    }

    /*
     * Allocate and return a pointer to a new GC object with its |slots|
     * pointer pre-filled. Returns nullptr if the Nursery is full.
     */
    JSObject* allocateObject(JSContext* cx, size_t size, size_t numDynamic, const js::Class* clasp);

    /* Allocate a buffer for a given zone, using the nursery if possible. */
    void* allocateBuffer(JS::Zone* zone, size_t nbytes);

    /*
     * Allocate a buffer for a given object, using the nursery if possible and
     * obj is in the nursery.
     */
    void* allocateBuffer(JSObject* obj, size_t nbytes);

    /* Resize an existing object buffer. */
    void* reallocateBuffer(JSObject* obj, void* oldBuffer,
                           size_t oldBytes, size_t newBytes);

    /* Free an object buffer. */
    void freeBuffer(void* buffer);

    /* The maximum number of bytes allowed to reside in nursery buffers. */
    static const size_t MaxNurseryBufferSize = 1024;

    /* Do a minor collection. */
    void collect(JSRuntime* rt, JS::gcreason::Reason reason);

    /*
     * Check if the thing at |*ref| in the Nursery has been forwarded. If so,
     * sets |*ref| to the new location of the object and returns true. Otherwise
     * returns false and leaves |*ref| unset.
     */
    MOZ_ALWAYS_INLINE MOZ_MUST_USE bool getForwardedPointer(JSObject** ref) const;

    /* Forward a slots/elements pointer stored in an Ion frame. */
    void forwardBufferPointer(HeapSlot** pSlotsElems);

    void maybeSetForwardingPointer(JSTracer* trc, void* oldData, void* newData, bool direct) {
        if (trc->isTenuringTracer() && isInside(oldData))
            setForwardingPointer(oldData, newData, direct);
    }

    /* Mark a malloced buffer as no longer needing to be freed. */
    void removeMallocedBuffer(void* buffer) {
        mallocedBuffers.remove(buffer);
    }

    void waitBackgroundFreeEnd();

    MOZ_MUST_USE bool addedUniqueIdToCell(gc::Cell* cell) {
        if (!IsInsideNursery(cell) || !isEnabled())
            return true;
        MOZ_ASSERT(cellsWithUid_.initialized());
        MOZ_ASSERT(!cellsWithUid_.has(cell));
        return cellsWithUid_.put(cell);
    }

    using SweepThunk = void (*)(void *data);
    void queueSweepAction(SweepThunk thunk, void* data);

    MOZ_MUST_USE bool queueDictionaryModeObjectToSweep(NativeObject* obj);

    size_t sizeOfHeapCommitted() const {
        return numChunks() * gc::ChunkSize;
    }
    size_t sizeOfMallocedBuffers(mozilla::MallocSizeOf mallocSizeOf) const {
        size_t total = 0;
        for (MallocedBuffersSet::Range r = mallocedBuffers.all(); !r.empty(); r.popFront())
            total += mallocSizeOf(r.front());
        total += mallocedBuffers.sizeOfExcludingThis(mallocSizeOf);
        return total;
    }

    // The number of bytes from the start position to the end of the nursery.
    size_t spaceToEnd() const;

    // Free space remaining, not counting chunk trailers.
    MOZ_ALWAYS_INLINE size_t freeSpace() const {
        MOZ_ASSERT(currentEnd_ - position_ <= NurseryChunkUsableSize);
        return (currentEnd_ - position_) +
               (numChunks() - currentChunk_ - 1) * NurseryChunkUsableSize;
    }

    /* Print total profile times on shutdown. */
    void printTotalProfileTimes();

  private:
    /* The amount of space in the mapped nursery available to allocations. */
    static const size_t NurseryChunkUsableSize = gc::ChunkSize - sizeof(gc::ChunkTrailer);

    struct NurseryChunk {
        char data[NurseryChunkUsableSize];
        gc::ChunkTrailer trailer;
        static NurseryChunk* fromChunk(gc::Chunk* chunk);
        void init(JSRuntime* rt);
        void poisonAndInit(JSRuntime* rt, uint8_t poison);
        uintptr_t start() const { return uintptr_t(&data); }
        uintptr_t end() const { return uintptr_t(&trailer); }
        gc::Chunk* toChunk(JSRuntime* rt);
    };
    static_assert(sizeof(NurseryChunk) == gc::ChunkSize,
                  "Nursery chunk size must match gc::Chunk size.");

    /*
     * The start and end pointers are stored under the runtime so that we can
     * inline the isInsideNursery check into embedder code. Use the start()
     * and heapEnd() functions to access these values.
     */
    JSRuntime* runtime_;

    /* Vector of allocated chunks to allocate from. */
    Vector<NurseryChunk*, 0, SystemAllocPolicy> chunks_;

    /* Pointer to the first unallocated byte in the nursery. */
    uintptr_t position_;

    /* Pointer to the logical start of the Nursery. */
    unsigned currentStartChunk_;
    uintptr_t currentStartPosition_;

    /* Pointer to the last byte of space in the current chunk. */
    uintptr_t currentEnd_;

    /* The index of the chunk that is currently being allocated from. */
    unsigned currentChunk_;

    /* Maximum number of chunks to allocate for the nursery. */
    unsigned maxNurseryChunks_;

    /* Promotion rate for the previous minor collection. */
    double previousPromotionRate_;

    /* Report minor collections taking at least this many us, if enabled. */
    int64_t profileThreshold_;
    bool enableProfiling_;

    /* Report ObjectGroups with at lest this many instances tenured. */
    int64_t reportTenurings_;

    /* Profiling data. */

    enum class ProfileKey
    {
#define DEFINE_TIME_KEY(name, text)                                           \
        name,
        FOR_EACH_NURSERY_PROFILE_TIME(DEFINE_TIME_KEY)
#undef DEFINE_TIME_KEY
        KeyCount
    };

    using ProfileTimes = mozilla::EnumeratedArray<ProfileKey, ProfileKey::KeyCount, int64_t>;

    ProfileTimes startTimes_;
    ProfileTimes profileTimes_;
    ProfileTimes totalTimes_;
    uint64_t minorGcCount_;

    /*
     * The set of externally malloced buffers potentially kept live by objects
     * stored in the nursery. Any external buffers that do not belong to a
     * tenured thing at the end of a minor GC must be freed.
     */
    typedef HashSet<void*, PointerHasher<void*, 3>, SystemAllocPolicy> MallocedBuffersSet;
    MallocedBuffersSet mallocedBuffers;

    /* A task structure used to free the malloced bufers on a background thread. */
    struct FreeMallocedBuffersTask;
    FreeMallocedBuffersTask* freeMallocedBuffersTask;

    /*
     * During a collection most hoisted slot and element buffers indicate their
     * new location with a forwarding pointer at the base. This does not work
     * for buffers whose length is less than pointer width, or when different
     * buffers might overlap each other. For these, an entry in the following
     * table is used.
     */
    typedef HashMap<void*, void*, PointerHasher<void*, 1>, SystemAllocPolicy> ForwardedBufferMap;
    ForwardedBufferMap forwardedBuffers;

    /*
     * When we assign a unique id to cell in the nursery, that almost always
     * means that the cell will be in a hash table, and thus, held live,
     * automatically moving the uid from the nursery to its new home in
     * tenured. It is possible, if rare, for an object that acquired a uid to
     * be dead before the next collection, in which case we need to know to
     * remove it when we sweep.
     *
     * Note: we store the pointers as Cell* here, resulting in an ugly cast in
     *       sweep. This is because this structure is used to help implement
     *       stable object hashing and we have to break the cycle somehow.
     */
    using CellsWithUniqueIdSet = HashSet<gc::Cell*, PointerHasher<gc::Cell*, 3>, SystemAllocPolicy>;
    CellsWithUniqueIdSet cellsWithUid_;

    struct SweepAction;
    SweepAction* sweepActions_;

    using NativeObjectVector = Vector<NativeObject*, 0, SystemAllocPolicy>;
    NativeObjectVector dictionaryModeObjects_;

    NurseryChunk* allocChunk();

    NurseryChunk& chunk(unsigned index) const {
        return *chunks_[index];
    }

    void setCurrentChunk(unsigned chunkno);
    void setStartPosition();

    void updateNumChunks(unsigned newCount);
    void updateNumChunksLocked(unsigned newCount,
                               gc::AutoMaybeStartBackgroundAllocation& maybeBgAlloc,
                               AutoLockGC& lock);

    MOZ_ALWAYS_INLINE uintptr_t allocationEnd() const {
        MOZ_ASSERT(numChunks() > 0);
        return chunks_.back()->end();
    }

    MOZ_ALWAYS_INLINE uintptr_t currentEnd() const {
        MOZ_ASSERT(runtime_);
        MOZ_ASSERT(currentEnd_ == chunk(currentChunk_).end());
        return currentEnd_;
    }
    void* addressOfCurrentEnd() const {
        MOZ_ASSERT(runtime_);
        return (void*)&currentEnd_;
    }

    uintptr_t position() const { return position_; }
    void* addressOfPosition() const { return (void*)&position_; }

    JSRuntime* runtime() const { return runtime_; }

    /* Allocates a new GC thing from the tenured generation during minor GC. */
    gc::TenuredCell* allocateFromTenured(JS::Zone* zone, gc::AllocKind thingKind);

    /* Common internal allocator function. */
    void* allocate(size_t size);

    double doCollection(JSRuntime* rt, JS::gcreason::Reason reason,
                        gc::TenureCountCache& tenureCounts);

    /*
     * Move the object at |src| in the Nursery to an already-allocated cell
     * |dst| in Tenured.
     */
    void collectToFixedPoint(TenuringTracer& trc, gc::TenureCountCache& tenureCounts);

    /* Handle relocation of slots/elements pointers stored in Ion frames. */
    void setForwardingPointer(void* oldData, void* newData, bool direct);

    void setSlotsForwardingPointer(HeapSlot* oldSlots, HeapSlot* newSlots, uint32_t nslots);
    void setElementsForwardingPointer(ObjectElements* oldHeader, ObjectElements* newHeader,
                                      uint32_t nelems);

    /* Free malloced pointers owned by freed things in the nursery. */
    void freeMallocedBuffers();

    /*
     * Frees all non-live nursery-allocated things at the end of a minor
     * collection.
     */
    void sweep();

    void runSweepActions();
    void sweepDictionaryModeObjects();

    /* Change the allocable space provided by the nursery. */
    void maybeResizeNursery(JS::gcreason::Reason reason, double promotionRate);
    void growAllocableSpace();
    void shrinkAllocableSpace();
    void minimizeAllocableSpace();

    /* Profile recording and printing. */
    void startProfile(ProfileKey key);
    void endProfile(ProfileKey key);
    void maybeStartProfile(ProfileKey key);
    void maybeEndProfile(ProfileKey key);
    static void printProfileHeader();
    static void printProfileTimes(const ProfileTimes& times);

    friend class TenuringTracer;
    friend class gc::MinorCollectionTracer;
    friend class jit::MacroAssembler;
};

} /* namespace js */

#endif /* gc_Nursery_h */