diff --git a/circleci.sh b/circleci.sh index f917fc985a3..f66206eed96 100755 --- a/circleci.sh +++ b/circleci.sh @@ -87,6 +87,9 @@ setup_repos() fi done + # enable debug GC logs + cp -f gc.d ../druntime/src/gc/impl/conservative/gc.d + # load environment for bootstrap compiler source "$(CURL_USER_AGENT=\"$CURL_USER_AGENT\" bash ~/dlang/install.sh dmd-$HOST_DMD_VER --activate)" @@ -102,6 +105,7 @@ style_lint() source "$(CURL_USER_AGENT=\"$CURL_USER_AGENT\" bash ~/dlang/install.sh dmd-$DSCANNER_DMD_VER --activate)" make -f posix.mak style_lint DUB=$DUB BUILD=$BUILD + cat "gcx.log" } # run unittest with coverage diff --git a/gc.d b/gc.d new file mode 100644 index 00000000000..110b32f33bb --- /dev/null +++ b/gc.d @@ -0,0 +1,3450 @@ +/** + * Contains the garbage collector implementation. + * + * Copyright: Copyright Digital Mars 2001 - 2016. + * License: $(WEB www.boost.org/LICENSE_1_0.txt, Boost License 1.0). + * Authors: Walter Bright, David Friedman, Sean Kelly + */ + +/* Copyright Digital Mars 2005 - 2016. + * Distributed under the Boost Software License, Version 1.0. + * (See accompanying file LICENSE or copy at + * http://www.boost.org/LICENSE_1_0.txt) + */ +module gc.impl.conservative.gc; + +// D Programming Language Garbage Collector implementation + +/************** Debugging ***************************/ + +//debug = PRINTF; // turn on printf's +//debug = COLLECT_PRINTF; // turn on printf's +//debug = PRINTF_TO_FILE; // redirect printf's ouptut to file "gcx.log" +//debug = LOGGING; // log allocations / frees +//debug = MEMSTOMP; // stomp on memory +//debug = SENTINEL; // add underrun/overrrun protection + // NOTE: this needs to be enabled globally in the makefiles + // (-debug=SENTINEL) to pass druntime's unittests. +//debug = PTRCHECK; // more pointer checking +//debug = PTRCHECK2; // thorough but slow pointer checking +//debug = INVARIANT; // enable invariants +//debug = PROFILE_API; // profile API calls for config.profile > 1 + +/*************** Configuration *********************/ + +version = STACKGROWSDOWN; // growing the stack means subtracting from the stack pointer + // (use for Intel X86 CPUs) + // else growing the stack means adding to the stack pointer + +/***************************************************/ + +import gc.bits; +import gc.os; +import gc.config; +import gc.gcinterface; + +import rt.util.container.treap; + +import cstdlib = core.stdc.stdlib : calloc, free, malloc, realloc; +import core.stdc.string : memcpy, memset, memmove; +import core.bitop; +import core.thread; +static import core.memory; + +version (GNU) import gcc.builtins; + +debug = PRINTF; +debug = COLLECT_PRINTF; +debug = PRINTF_TO_FILE; + +debug (PRINTF_TO_FILE) import core.stdc.stdio : sprintf, fprintf, fopen, fflush, FILE; +else import core.stdc.stdio : sprintf, printf; // needed to output profiling results + +import core.time; +alias currTime = MonoTime.currTime; + +debug(PRINTF_TO_FILE) +{ + private __gshared MonoTime gcStartTick; + private __gshared FILE* gcx_fh; + + private int printf(ARGS...)(const char* fmt, ARGS args) nothrow + { + if (!gcx_fh) + gcx_fh = fopen("gcx.log", "w"); + if (!gcx_fh) + return 0; + + int len; + if (MonoTime.ticksPerSecond == 0) + { + len = fprintf(gcx_fh, "before init: "); + } + else + { + if (gcStartTick == MonoTime.init) + gcStartTick = MonoTime.currTime; + immutable timeElapsed = MonoTime.currTime - gcStartTick; + immutable secondsAsDouble = timeElapsed.total!"hnsecs" / cast(double)convert!("seconds", "hnsecs")(1); + len = fprintf(gcx_fh, "%10.6lf: ", secondsAsDouble); + } + len += fprintf(gcx_fh, fmt, args); + fflush(gcx_fh); + return len; + } +} + +debug(PRINTF) void printFreeInfo(Pool* pool) nothrow +{ + uint nReallyFree; + foreach(i; 0..pool.npages) { + if(pool.pagetable[i] >= B_FREE) nReallyFree++; + } + + printf("Pool %p: %d really free, %d supposedly free\n", pool, nReallyFree, pool.freepages); +} + +// Track total time spent preparing for GC, +// marking, sweeping and recovering pages. +__gshared Duration prepTime; +__gshared Duration markTime; +__gshared Duration sweepTime; +__gshared Duration recoverTime; +__gshared Duration maxPauseTime; +__gshared size_t numCollections; +__gshared size_t maxPoolMemory; + +__gshared long numMallocs; +__gshared long numFrees; +__gshared long numReallocs; +__gshared long numExtends; +__gshared long numOthers; +__gshared long mallocTime; // using ticks instead of MonoTime for better performance +__gshared long freeTime; +__gshared long reallocTime; +__gshared long extendTime; +__gshared long otherTime; +__gshared long lockTime; + +private +{ + extern (C) + { + // to allow compilation of this module without access to the rt package, + // make these functions available from rt.lifetime + void rt_finalizeFromGC(void* p, size_t size, uint attr) nothrow; + int rt_hasFinalizerInSegment(void* p, size_t size, uint attr, in void[] segment) nothrow; + + // Declared as an extern instead of importing core.exception + // to avoid inlining - see issue 13725. + void onInvalidMemoryOperationError() @nogc nothrow; + void onOutOfMemoryErrorNoGC() @nogc nothrow; + } + + enum + { + OPFAIL = ~cast(size_t)0 + } +} + + +alias GC gc_t; + + +/* ======================= Leak Detector =========================== */ + + +debug (LOGGING) +{ + struct Log + { + void* p; + size_t size; + size_t line; + char* file; + void* parent; + + void print() nothrow + { + printf(" p = %p, size = %zd, parent = %p ", p, size, parent); + if (file) + { + printf("%s(%u)", file, line); + } + printf("\n"); + } + } + + + struct LogArray + { + size_t dim; + size_t allocdim; + Log *data; + + void Dtor() nothrow + { + if (data) + cstdlib.free(data); + data = null; + } + + void reserve(size_t nentries) nothrow + { + assert(dim <= allocdim); + if (allocdim - dim < nentries) + { + allocdim = (dim + nentries) * 2; + assert(dim + nentries <= allocdim); + if (!data) + { + data = cast(Log*)cstdlib.malloc(allocdim * Log.sizeof); + if (!data && allocdim) + onOutOfMemoryErrorNoGC(); + } + else + { Log *newdata; + + newdata = cast(Log*)cstdlib.malloc(allocdim * Log.sizeof); + if (!newdata && allocdim) + onOutOfMemoryErrorNoGC(); + memcpy(newdata, data, dim * Log.sizeof); + cstdlib.free(data); + data = newdata; + } + } + } + + + void push(Log log) nothrow + { + reserve(1); + data[dim++] = log; + } + + void remove(size_t i) nothrow + { + memmove(data + i, data + i + 1, (dim - i) * Log.sizeof); + dim--; + } + + + size_t find(void *p) nothrow + { + for (size_t i = 0; i < dim; i++) + { + if (data[i].p == p) + return i; + } + return OPFAIL; // not found + } + + + void copy(LogArray *from) nothrow + { + reserve(from.dim - dim); + assert(from.dim <= allocdim); + memcpy(data, from.data, from.dim * Log.sizeof); + dim = from.dim; + } + } +} + + +/* ============================ GC =============================== */ + +class ConservativeGC : GC +{ + // For passing to debug code (not thread safe) + __gshared size_t line; + __gshared char* file; + + Gcx *gcx; // implementation + + import core.internal.spinlock; + static gcLock = shared(AlignedSpinLock)(SpinLock.Contention.lengthy); + static bool _inFinalizer; + + // lock GC, throw InvalidMemoryOperationError on recursive locking during finalization + static void lockNR() @nogc nothrow + { + if (_inFinalizer) + onInvalidMemoryOperationError(); + gcLock.lock(); + } + + + static void initialize(ref GC gc) + { + import core.stdc.string: memcpy; + + if(config.gc != "conservative") + return; + + auto p = cstdlib.malloc(__traits(classInstanceSize,ConservativeGC)); + + if(!p) + onOutOfMemoryErrorNoGC(); + + auto init = typeid(ConservativeGC).initializer(); + assert(init.length == __traits(classInstanceSize, ConservativeGC)); + auto instance = cast(ConservativeGC) memcpy(p, init.ptr, init.length); + instance.__ctor(); + + gc = instance; + } + + + static void finalize(ref GC gc) + { + if(config.gc != "conservative") + return; + + auto instance = cast(ConservativeGC) gc; + instance.Dtor(); + cstdlib.free(cast(void*)instance); + } + + + this() + { + //config is assumed to have already been initialized + + gcx = cast(Gcx*)cstdlib.calloc(1, Gcx.sizeof); + if (!gcx) + onOutOfMemoryErrorNoGC(); + gcx.initialize(); + + if (config.initReserve) + gcx.reserve(config.initReserve << 20); + if (config.disable) + gcx.disabled++; + } + + + void Dtor() + { + version (linux) + { + //debug(PRINTF) printf("Thread %x ", pthread_self()); + //debug(PRINTF) printf("GC.Dtor()\n"); + } + + if (gcx) + { + gcx.Dtor(); + cstdlib.free(gcx); + gcx = null; + } + } + + + void enable() + { + static void go(Gcx* gcx) nothrow + { + assert(gcx.disabled > 0); + gcx.disabled--; + } + runLocked!(go, otherTime, numOthers)(gcx); + } + + + void disable() + { + static void go(Gcx* gcx) nothrow + { + gcx.disabled++; + } + runLocked!(go, otherTime, numOthers)(gcx); + } + + + auto runLocked(alias func, Args...)(auto ref Args args) + { + debug(PROFILE_API) immutable tm = (config.profile > 1 ? currTime.ticks : 0); + lockNR(); + scope (failure) gcLock.unlock(); + debug(PROFILE_API) immutable tm2 = (config.profile > 1 ? currTime.ticks : 0); + + static if (is(typeof(func(args)) == void)) + func(args); + else + auto res = func(args); + + debug(PROFILE_API) if (config.profile > 1) + lockTime += tm2 - tm; + gcLock.unlock(); + + static if (!is(typeof(func(args)) == void)) + return res; + } + + + auto runLocked(alias func, alias time, alias count, Args...)(auto ref Args args) + { + debug(PROFILE_API) immutable tm = (config.profile > 1 ? currTime.ticks : 0); + lockNR(); + scope (failure) gcLock.unlock(); + debug(PROFILE_API) immutable tm2 = (config.profile > 1 ? currTime.ticks : 0); + + static if (is(typeof(func(args)) == void)) + func(args); + else + auto res = func(args); + + debug(PROFILE_API) if (config.profile > 1) + { + count++; + immutable now = currTime.ticks; + lockTime += tm2 - tm; + time += now - tm2; + } + gcLock.unlock(); + + static if (!is(typeof(func(args)) == void)) + return res; + } + + + uint getAttr(void* p) nothrow + { + if (!p) + { + return 0; + } + + static uint go(Gcx* gcx, void* p) nothrow + { + Pool* pool = gcx.findPool(p); + uint oldb = 0; + + if (pool) + { + p = sentinel_sub(p); + auto biti = cast(size_t)(p - pool.baseAddr) >> pool.shiftBy; + + oldb = pool.getBits(biti); + } + return oldb; + } + + return runLocked!(go, otherTime, numOthers)(gcx, p); + } + + + uint setAttr(void* p, uint mask) nothrow + { + if (!p) + { + return 0; + } + + static uint go(Gcx* gcx, void* p, uint mask) nothrow + { + Pool* pool = gcx.findPool(p); + uint oldb = 0; + + if (pool) + { + p = sentinel_sub(p); + auto biti = cast(size_t)(p - pool.baseAddr) >> pool.shiftBy; + + oldb = pool.getBits(biti); + pool.setBits(biti, mask); + } + return oldb; + } + + return runLocked!(go, otherTime, numOthers)(gcx, p, mask); + } + + + uint clrAttr(void* p, uint mask) nothrow + { + if (!p) + { + return 0; + } + + static uint go(Gcx* gcx, void* p, uint mask) nothrow + { + Pool* pool = gcx.findPool(p); + uint oldb = 0; + + if (pool) + { + p = sentinel_sub(p); + auto biti = cast(size_t)(p - pool.baseAddr) >> pool.shiftBy; + + oldb = pool.getBits(biti); + pool.clrBits(biti, mask); + } + return oldb; + } + + return runLocked!(go, otherTime, numOthers)(gcx, p, mask); + } + + + void *malloc(size_t size, uint bits, const TypeInfo ti) nothrow + { + if (!size) + { + return null; + } + + size_t localAllocSize = void; + + auto p = runLocked!(mallocNoSync, mallocTime, numMallocs)(size, bits, localAllocSize, ti); + + if (!(bits & BlkAttr.NO_SCAN)) + { + memset(p + size, 0, localAllocSize - size); + } + + return p; + } + + + // + // + // + private void *mallocNoSync(size_t size, uint bits, ref size_t alloc_size, const TypeInfo ti = null) nothrow + { + assert(size != 0); + + //debug(PRINTF) printf("GC::malloc(size = %d, gcx = %p)\n", size, gcx); + assert(gcx); + //debug(PRINTF) printf("gcx.self = %x, pthread_self() = %x\n", gcx.self, pthread_self()); + + auto p = gcx.alloc(size + SENTINEL_EXTRA, alloc_size, bits); + if (!p) + onOutOfMemoryErrorNoGC(); + + debug (SENTINEL) + { + p = sentinel_add(p); + sentinel_init(p, size); + alloc_size = size; + } + gcx.log_malloc(p, size); + + return p; + } + + + BlkInfo qalloc( size_t size, uint bits, const TypeInfo ti) nothrow + { + + if (!size) + { + return BlkInfo.init; + } + + BlkInfo retval; + + retval.base = runLocked!(mallocNoSync, mallocTime, numMallocs)(size, bits, retval.size, ti); + + if (!(bits & BlkAttr.NO_SCAN)) + { + memset(retval.base + size, 0, retval.size - size); + } + + retval.attr = bits; + return retval; + } + + + void *calloc(size_t size, uint bits, const TypeInfo ti) nothrow + { + if (!size) + { + return null; + } + + size_t localAllocSize = void; + + auto p = runLocked!(mallocNoSync, mallocTime, numMallocs)(size, bits, localAllocSize, ti); + + memset(p, 0, size); + if (!(bits & BlkAttr.NO_SCAN)) + { + memset(p + size, 0, localAllocSize - size); + } + + return p; + } + + + void *realloc(void *p, size_t size, uint bits, const TypeInfo ti) nothrow + { + size_t localAllocSize = void; + auto oldp = p; + + p = runLocked!(reallocNoSync, mallocTime, numMallocs)(p, size, bits, localAllocSize, ti); + + if (p !is oldp && !(bits & BlkAttr.NO_SCAN)) + { + memset(p + size, 0, localAllocSize - size); + } + + return p; + } + + + // + // bits will be set to the resulting bits of the new block + // + private void *reallocNoSync(void *p, size_t size, ref uint bits, ref size_t alloc_size, const TypeInfo ti = null) nothrow + { + if (!size) + { if (p) + { freeNoSync(p); + p = null; + } + alloc_size = 0; + } + else if (!p) + { + p = mallocNoSync(size, bits, alloc_size, ti); + } + else + { void *p2; + size_t psize; + + //debug(PRINTF) printf("GC::realloc(p = %p, size = %zu)\n", p, size); + debug (SENTINEL) + { + sentinel_Invariant(p); + psize = *sentinel_size(p); + if (psize != size) + { + if (psize) + { + Pool *pool = gcx.findPool(p); + + if (pool) + { + auto biti = cast(size_t)(sentinel_sub(p) - pool.baseAddr) >> pool.shiftBy; + + if (bits) + { + pool.clrBits(biti, ~BlkAttr.NONE); + pool.setBits(biti, bits); + } + else + { + bits = pool.getBits(biti); + } + } + } + p2 = mallocNoSync(size, bits, alloc_size, ti); + if (psize < size) + size = psize; + //debug(PRINTF) printf("\tcopying %d bytes\n",size); + memcpy(p2, p, size); + p = p2; + } + } + else + { + auto pool = gcx.findPool(p); + if (pool.isLargeObject) + { + auto lpool = cast(LargeObjectPool*) pool; + psize = lpool.getSize(p); // get allocated size + + if (size <= PAGESIZE / 2) + goto Lmalloc; // switching from large object pool to small object pool + + auto psz = psize / PAGESIZE; + auto newsz = (size + PAGESIZE - 1) / PAGESIZE; + if (newsz == psz) + { + alloc_size = psize; + return p; + } + + auto pagenum = lpool.pagenumOf(p); + + if (newsz < psz) + { // Shrink in place + debug (MEMSTOMP) memset(p + size, 0xF2, psize - size); + lpool.freePages(pagenum + newsz, psz - newsz); + } + else if (pagenum + newsz <= pool.npages) + { // Attempt to expand in place + foreach (binsz; lpool.pagetable[pagenum + psz .. pagenum + newsz]) + if (binsz != B_FREE) + goto Lmalloc; + + debug (MEMSTOMP) memset(p + psize, 0xF0, size - psize); + debug(PRINTF) printFreeInfo(pool); + memset(&lpool.pagetable[pagenum + psz], B_PAGEPLUS, newsz - psz); + gcx.usedLargePages += newsz - psz; + lpool.freepages -= (newsz - psz); + debug(PRINTF) printFreeInfo(pool); + } + else + goto Lmalloc; // does not fit into current pool + + lpool.updateOffsets(pagenum); + if (bits) + { + immutable biti = cast(size_t)(p - pool.baseAddr) >> pool.shiftBy; + pool.clrBits(biti, ~BlkAttr.NONE); + pool.setBits(biti, bits); + } + alloc_size = newsz * PAGESIZE; + return p; + } + + psize = (cast(SmallObjectPool*) pool).getSize(p); // get allocated size + if (psize < size || // if new size is bigger + psize > size * 2) // or less than half + { + Lmalloc: + if (psize && pool) + { + auto biti = cast(size_t)(p - pool.baseAddr) >> pool.shiftBy; + + if (bits) + { + pool.clrBits(biti, ~BlkAttr.NONE); + pool.setBits(biti, bits); + } + else + { + bits = pool.getBits(biti); + } + } + p2 = mallocNoSync(size, bits, alloc_size, ti); + if (psize < size) + size = psize; + //debug(PRINTF) printf("\tcopying %d bytes\n",size); + memcpy(p2, p, size); + p = p2; + } + else + alloc_size = psize; + } + } + return p; + } + + + size_t extend(void* p, size_t minsize, size_t maxsize, const TypeInfo ti) nothrow + { + return runLocked!(extendNoSync, extendTime, numExtends)(p, minsize, maxsize, ti); + } + + + // + // + // + private size_t extendNoSync(void* p, size_t minsize, size_t maxsize, const TypeInfo ti = null) nothrow + in + { + assert(minsize <= maxsize); + } + do + { + //debug(PRINTF) printf("GC::extend(p = %p, minsize = %zu, maxsize = %zu)\n", p, minsize, maxsize); + debug (SENTINEL) + { + return 0; + } + else + { + auto pool = gcx.findPool(p); + if (!pool || !pool.isLargeObject) + return 0; + + auto lpool = cast(LargeObjectPool*) pool; + auto psize = lpool.getSize(p); // get allocated size + if (psize < PAGESIZE) + return 0; // cannot extend buckets + + auto psz = psize / PAGESIZE; + auto minsz = (minsize + PAGESIZE - 1) / PAGESIZE; + auto maxsz = (maxsize + PAGESIZE - 1) / PAGESIZE; + + auto pagenum = lpool.pagenumOf(p); + + size_t sz; + for (sz = 0; sz < maxsz; sz++) + { + auto i = pagenum + psz + sz; + if (i == lpool.npages) + break; + if (lpool.pagetable[i] != B_FREE) + { if (sz < minsz) + return 0; + break; + } + } + if (sz < minsz) + return 0; + debug (MEMSTOMP) memset(pool.baseAddr + (pagenum + psz) * PAGESIZE, 0xF0, sz * PAGESIZE); + memset(lpool.pagetable + pagenum + psz, B_PAGEPLUS, sz); + lpool.updateOffsets(pagenum); + lpool.freepages -= sz; + gcx.usedLargePages += sz; + return (psz + sz) * PAGESIZE; + } + } + + + size_t reserve(size_t size) nothrow + { + if (!size) + { + return 0; + } + + return runLocked!(reserveNoSync, otherTime, numOthers)(size); + } + + + // + // + // + private size_t reserveNoSync(size_t size) nothrow + { + assert(size != 0); + assert(gcx); + + return gcx.reserve(size); + } + + + void free(void *p) nothrow @nogc + { + if (!p || _inFinalizer) + { + return; + } + + return runLocked!(freeNoSync, freeTime, numFrees)(p); + } + + + // + // + // + private void freeNoSync(void *p) nothrow @nogc + { + debug(PRINTF) printf("Freeing %p\n", cast(size_t) p); + assert (p); + + Pool* pool; + size_t pagenum; + Bins bin; + size_t biti; + + // Find which page it is in + pool = gcx.findPool(p); + if (!pool) // if not one of ours + return; // ignore + + pagenum = pool.pagenumOf(p); + + debug(PRINTF) printf("pool base = %p, PAGENUM = %d of %d, bin = %d\n", pool.baseAddr, pagenum, pool.npages, pool.pagetable[pagenum]); + debug(PRINTF) if(pool.isLargeObject) printf("Block size = %d\n", pool.bPageOffsets[pagenum]); + + bin = cast(Bins)pool.pagetable[pagenum]; + + // Verify that the pointer is at the beginning of a block, + // no action should be taken if p is an interior pointer + if (bin > B_PAGE) // B_PAGEPLUS or B_FREE + return; + if ((sentinel_sub(p) - pool.baseAddr) & (binsize[bin] - 1)) + return; + + sentinel_Invariant(p); + p = sentinel_sub(p); + biti = cast(size_t)(p - pool.baseAddr) >> pool.shiftBy; + + pool.clrBits(biti, ~BlkAttr.NONE); + + if (pool.isLargeObject) // if large alloc + { + assert(bin == B_PAGE); + auto lpool = cast(LargeObjectPool*) pool; + + // Free pages + size_t npages = lpool.bPageOffsets[pagenum]; + debug (MEMSTOMP) memset(p, 0xF2, npages * PAGESIZE); + lpool.freePages(pagenum, npages); + } + else + { // Add to free list + List *list = cast(List*)p; + + debug (MEMSTOMP) memset(p, 0xF2, binsize[bin]); + + list.next = gcx.bucket[bin]; + list.pool = pool; + gcx.bucket[bin] = list; + } + + gcx.log_free(sentinel_add(p)); + } + + + void* addrOf(void *p) nothrow @nogc + { + if (!p) + { + return null; + } + + return runLocked!(addrOfNoSync, otherTime, numOthers)(p); + } + + + // + // + // + void* addrOfNoSync(void *p) nothrow @nogc + { + if (!p) + { + return null; + } + + auto q = gcx.findBase(p); + if (q) + q = sentinel_add(q); + return q; + } + + + size_t sizeOf(void *p) nothrow @nogc + { + if (!p) + { + return 0; + } + + return runLocked!(sizeOfNoSync, otherTime, numOthers)(p); + } + + + // + // + // + private size_t sizeOfNoSync(void *p) nothrow @nogc + { + assert (p); + + debug (SENTINEL) + { + p = sentinel_sub(p); + size_t size = gcx.findSize(p); + + // Check for interior pointer + // This depends on: + // 1) size is a power of 2 for less than PAGESIZE values + // 2) base of memory pool is aligned on PAGESIZE boundary + if (cast(size_t)p & (size - 1) & (PAGESIZE - 1)) + size = 0; + return size ? size - SENTINEL_EXTRA : 0; + } + else + { + size_t size = gcx.findSize(p); + + // Check for interior pointer + // This depends on: + // 1) size is a power of 2 for less than PAGESIZE values + // 2) base of memory pool is aligned on PAGESIZE boundary + if (cast(size_t)p & (size - 1) & (PAGESIZE - 1)) + return 0; + return size; + } + } + + + BlkInfo query(void *p) nothrow + { + if (!p) + { + BlkInfo i; + return i; + } + + return runLocked!(queryNoSync, otherTime, numOthers)(p); + } + + // + // + // + BlkInfo queryNoSync(void *p) nothrow + { + assert(p); + + BlkInfo info = gcx.getInfo(p); + debug(SENTINEL) + { + if (info.base) + { + info.base = sentinel_add(info.base); + info.size = *sentinel_size(info.base); + } + } + return info; + } + + + /** + * Verify that pointer p: + * 1) belongs to this memory pool + * 2) points to the start of an allocated piece of memory + * 3) is not on a free list + */ + void check(void *p) nothrow + { + if (!p) + { + return; + } + + return runLocked!(checkNoSync, otherTime, numOthers)(p); + } + + + // + // + // + private void checkNoSync(void *p) nothrow + { + assert(p); + + sentinel_Invariant(p); + debug (PTRCHECK) + { + Pool* pool; + size_t pagenum; + Bins bin; + size_t size; + + p = sentinel_sub(p); + pool = gcx.findPool(p); + assert(pool); + pagenum = pool.pagenumOf(p); + bin = cast(Bins)pool.pagetable[pagenum]; + assert(bin <= B_PAGE); + size = binsize[bin]; + assert((cast(size_t)p & (size - 1)) == 0); + + debug (PTRCHECK2) + { + if (bin < B_PAGE) + { + // Check that p is not on a free list + List *list; + + for (list = gcx.bucket[bin]; list; list = list.next) + { + assert(cast(void*)list != p); + } + } + } + } + } + + + void addRoot(void *p) nothrow @nogc + { + if (!p) + { + return; + } + + gcx.addRoot(p); + } + + + void removeRoot(void *p) nothrow @nogc + { + if (!p) + { + return; + } + + gcx.removeRoot(p); + } + + + @property RootIterator rootIter() @nogc + { + return &gcx.rootsApply; + } + + + void addRange(void *p, size_t sz, const TypeInfo ti = null) nothrow @nogc + { + if (!p || !sz) + { + return; + } + + gcx.addRange(p, p + sz, ti); + } + + + void removeRange(void *p) nothrow @nogc + { + if (!p) + { + return; + } + + gcx.removeRange(p); + } + + + @property RangeIterator rangeIter() @nogc + { + return &gcx.rangesApply; + } + + + void runFinalizers(in void[] segment) nothrow + { + static void go(Gcx* gcx, in void[] segment) nothrow + { + gcx.runFinalizers(segment); + } + return runLocked!(go, otherTime, numOthers)(gcx, segment); + } + + + bool inFinalizer() nothrow + { + return _inFinalizer; + } + + + void collect() nothrow + { + fullCollect(); + } + + + void collectNoStack() nothrow + { + fullCollectNoStack(); + } + + + /** + * Do full garbage collection. + * Return number of pages free'd. + */ + size_t fullCollect() nothrow + { + debug(PRINTF) printf("GC.fullCollect()\n"); + + // Since a finalizer could launch a new thread, we always need to lock + // when collecting. + static size_t go(Gcx* gcx) nothrow + { + return gcx.fullcollect(); + } + immutable result = runLocked!go(gcx); + + version (none) + { + GCStats stats; + + getStats(stats); + debug(PRINTF) printf("heapSize = %zx, freeSize = %zx\n", + stats.heapSize, stats.freeSize); + } + + gcx.log_collect(); + return result; + } + + + /** + * do full garbage collection ignoring roots + */ + void fullCollectNoStack() nothrow + { + // Since a finalizer could launch a new thread, we always need to lock + // when collecting. + static size_t go(Gcx* gcx) nothrow + { + return gcx.fullcollect(true); + } + runLocked!go(gcx); + } + + + void minimize() nothrow + { + static void go(Gcx* gcx) nothrow + { + gcx.minimize(); + } + runLocked!(go, otherTime, numOthers)(gcx); + } + + + core.memory.GC.Stats stats() nothrow + { + typeof(return) ret; + + runLocked!(getStatsNoSync, otherTime, numOthers)(ret); + + return ret; + } + + + // + // + // + private void getStatsNoSync(out core.memory.GC.Stats stats) nothrow + { + foreach (pool; gcx.pooltable[0 .. gcx.npools]) + { + foreach (bin; pool.pagetable[0 .. pool.npages]) + { + if (bin == B_FREE) + stats.freeSize += PAGESIZE; + else + stats.usedSize += PAGESIZE; + } + } + + size_t freeListSize; + foreach (n; 0 .. B_PAGE) + { + immutable sz = binsize[n]; + for (List *list = gcx.bucket[n]; list; list = list.next) + freeListSize += sz; + } + + stats.usedSize -= freeListSize; + stats.freeSize += freeListSize; + } +} + + +/* ============================ Gcx =============================== */ + +enum +{ PAGESIZE = 4096, + POOLSIZE = (4096*256), +} + + +enum +{ + B_16, + B_32, + B_64, + B_128, + B_256, + B_512, + B_1024, + B_2048, + B_PAGE, // start of large alloc + B_PAGEPLUS, // continuation of large alloc + B_FREE, // free page + B_MAX +} + + +alias ubyte Bins; + + +struct List +{ + List *next; + Pool *pool; +} + + +immutable uint[B_MAX] binsize = [ 16,32,64,128,256,512,1024,2048,4096 ]; +immutable size_t[B_MAX] notbinsize = [ ~(16-1),~(32-1),~(64-1),~(128-1),~(256-1), + ~(512-1),~(1024-1),~(2048-1),~(4096-1) ]; + +alias PageBits = GCBits.wordtype[PAGESIZE / 16 / GCBits.BITS_PER_WORD]; +static assert(PAGESIZE % (GCBits.BITS_PER_WORD * 16) == 0); + +private void set(ref PageBits bits, size_t i) @nogc pure nothrow +{ + assert(i < PageBits.sizeof * 8); + bts(bits.ptr, i); +} + +/* ============================ Gcx =============================== */ + +struct Gcx +{ + import core.internal.spinlock; + auto rootsLock = shared(AlignedSpinLock)(SpinLock.Contention.brief); + auto rangesLock = shared(AlignedSpinLock)(SpinLock.Contention.brief); + Treap!Root roots; + Treap!Range ranges; + + bool log; // turn on logging + debug(INVARIANT) bool initialized; + uint disabled; // turn off collections if >0 + + import gc.pooltable; + @property size_t npools() pure const nothrow { return pooltable.length; } + PoolTable!Pool pooltable; + + List*[B_PAGE] bucket; // free list for each small size + + // run a collection when reaching those thresholds (number of used pages) + float smallCollectThreshold, largeCollectThreshold; + uint usedSmallPages, usedLargePages; + // total number of mapped pages + uint mappedPages; + + void initialize() + { + (cast(byte*)&this)[0 .. Gcx.sizeof] = 0; + log_init(); + roots.initialize(); + ranges.initialize(); + smallCollectThreshold = largeCollectThreshold = 0.0f; + usedSmallPages = usedLargePages = 0; + mappedPages = 0; + //printf("gcx = %p, self = %x\n", &this, self); + debug(INVARIANT) initialized = true; + } + + + void Dtor() + { + if (config.profile) + { + printf("\tNumber of collections: %llu\n", cast(ulong)numCollections); + printf("\tTotal GC prep time: %lld milliseconds\n", + prepTime.total!("msecs")); + printf("\tTotal mark time: %lld milliseconds\n", + markTime.total!("msecs")); + printf("\tTotal sweep time: %lld milliseconds\n", + sweepTime.total!("msecs")); + printf("\tTotal page recovery time: %lld milliseconds\n", + recoverTime.total!("msecs")); + long maxPause = maxPauseTime.total!("msecs"); + printf("\tMax Pause Time: %lld milliseconds\n", maxPause); + long gcTime = (recoverTime + sweepTime + markTime + prepTime).total!("msecs"); + printf("\tGrand total GC time: %lld milliseconds\n", gcTime); + long pauseTime = (markTime + prepTime).total!("msecs"); + + char[30] apitxt; + apitxt[0] = 0; + debug(PROFILE_API) if (config.profile > 1) + { + static Duration toDuration(long dur) + { + return MonoTime(dur) - MonoTime(0); + } + + printf("\n"); + printf("\tmalloc: %llu calls, %lld ms\n", cast(ulong)numMallocs, toDuration(mallocTime).total!"msecs"); + printf("\trealloc: %llu calls, %lld ms\n", cast(ulong)numReallocs, toDuration(reallocTime).total!"msecs"); + printf("\tfree: %llu calls, %lld ms\n", cast(ulong)numFrees, toDuration(freeTime).total!"msecs"); + printf("\textend: %llu calls, %lld ms\n", cast(ulong)numExtends, toDuration(extendTime).total!"msecs"); + printf("\tother: %llu calls, %lld ms\n", cast(ulong)numOthers, toDuration(otherTime).total!"msecs"); + printf("\tlock time: %lld ms\n", toDuration(lockTime).total!"msecs"); + + long apiTime = mallocTime + reallocTime + freeTime + extendTime + otherTime + lockTime; + printf("\tGC API: %lld ms\n", toDuration(apiTime).total!"msecs"); + sprintf(apitxt.ptr, " API%5ld ms", toDuration(apiTime).total!"msecs"); + } + + printf("GC summary:%5lld MB,%5lld GC%5lld ms, Pauses%5lld ms <%5lld ms%s\n", + cast(long) maxPoolMemory >> 20, cast(ulong)numCollections, gcTime, + pauseTime, maxPause, apitxt.ptr); + } + + debug(INVARIANT) initialized = false; + + for (size_t i = 0; i < npools; i++) + { + Pool *pool = pooltable[i]; + mappedPages -= pool.npages; + pool.Dtor(); + cstdlib.free(pool); + } + assert(!mappedPages); + pooltable.Dtor(); + + roots.removeAll(); + ranges.removeAll(); + toscan.reset(); + } + + + void Invariant() const { } + + debug(INVARIANT) + invariant() + { + if (initialized) + { + //printf("Gcx.invariant(): this = %p\n", &this); + pooltable.Invariant(); + + rangesLock.lock(); + foreach (range; ranges) + { + assert(range.pbot); + assert(range.ptop); + assert(range.pbot <= range.ptop); + } + rangesLock.unlock(); + + for (size_t i = 0; i < B_PAGE; i++) + { + for (auto list = cast(List*)bucket[i]; list; list = list.next) + { + } + } + } + } + + + /** + * + */ + void addRoot(void *p) nothrow @nogc + { + rootsLock.lock(); + scope (failure) rootsLock.unlock(); + roots.insert(Root(p)); + rootsLock.unlock(); + } + + + /** + * + */ + void removeRoot(void *p) nothrow @nogc + { + rootsLock.lock(); + scope (failure) rootsLock.unlock(); + roots.remove(Root(p)); + rootsLock.unlock(); + } + + + /** + * + */ + int rootsApply(scope int delegate(ref Root) nothrow dg) nothrow + { + rootsLock.lock(); + scope (failure) rootsLock.unlock(); + auto ret = roots.opApply(dg); + rootsLock.unlock(); + return ret; + } + + + /** + * + */ + void addRange(void *pbot, void *ptop, const TypeInfo ti) nothrow @nogc + { + //debug(PRINTF) printf("Thread %x ", pthread_self()); + debug(PRINTF) printf("%p.Gcx::addRange(%p, %p)\n", &this, pbot, ptop); + rangesLock.lock(); + scope (failure) rangesLock.unlock(); + ranges.insert(Range(pbot, ptop)); + rangesLock.unlock(); + } + + + /** + * + */ + void removeRange(void *pbot) nothrow @nogc + { + //debug(PRINTF) printf("Thread %x ", pthread_self()); + debug(PRINTF) printf("Gcx.removeRange(%p)\n", pbot); + rangesLock.lock(); + scope (failure) rangesLock.unlock(); + ranges.remove(Range(pbot, pbot)); // only pbot is used, see Range.opCmp + rangesLock.unlock(); + + // debug(PRINTF) printf("Wrong thread\n"); + // This is a fatal error, but ignore it. + // The problem is that we can get a Close() call on a thread + // other than the one the range was allocated on. + //assert(zero); + } + + /** + * + */ + int rangesApply(scope int delegate(ref Range) nothrow dg) nothrow + { + rangesLock.lock(); + scope (failure) rangesLock.unlock(); + auto ret = ranges.opApply(dg); + rangesLock.unlock(); + return ret; + } + + + /** + * + */ + void runFinalizers(in void[] segment) nothrow + { + ConservativeGC._inFinalizer = true; + scope (failure) ConservativeGC._inFinalizer = false; + + foreach (pool; pooltable[0 .. npools]) + { + if (!pool.finals.nbits) continue; + + if (pool.isLargeObject) + { + auto lpool = cast(LargeObjectPool*) pool; + lpool.runFinalizers(segment); + } + else + { + auto spool = cast(SmallObjectPool*) pool; + spool.runFinalizers(segment); + } + } + ConservativeGC._inFinalizer = false; + } + + Pool* findPool(void* p) pure nothrow @nogc + { + return pooltable.findPool(p); + } + + /** + * Find base address of block containing pointer p. + * Returns null if not a gc'd pointer + */ + void* findBase(void *p) nothrow @nogc + { + Pool *pool; + + pool = findPool(p); + if (pool) + { + size_t offset = cast(size_t)(p - pool.baseAddr); + size_t pn = offset / PAGESIZE; + Bins bin = cast(Bins)pool.pagetable[pn]; + + // Adjust bit to be at start of allocated memory block + if (bin <= B_PAGE) + { + return pool.baseAddr + (offset & notbinsize[bin]); + } + else if (bin == B_PAGEPLUS) + { + auto pageOffset = pool.bPageOffsets[pn]; + offset -= pageOffset * PAGESIZE; + pn -= pageOffset; + + return pool.baseAddr + (offset & (offset.max ^ (PAGESIZE-1))); + } + else + { + // we are in a B_FREE page + assert(bin == B_FREE); + return null; + } + } + return null; + } + + + /** + * Find size of pointer p. + * Returns 0 if not a gc'd pointer + */ + size_t findSize(void *p) nothrow @nogc + { + Pool* pool = findPool(p); + if (pool) + return pool.slGetSize(p); + return 0; + } + + /** + * + */ + BlkInfo getInfo(void* p) nothrow + { + Pool* pool = findPool(p); + if (pool) + return pool.slGetInfo(p); + return BlkInfo(); + } + + /** + * Computes the bin table using CTFE. + */ + static byte[2049] ctfeBins() nothrow + { + byte[2049] ret; + size_t p = 0; + for (Bins b = B_16; b <= B_2048; b++) + for ( ; p <= binsize[b]; p++) + ret[p] = b; + + return ret; + } + + static const byte[2049] binTable = ctfeBins(); + + /** + * Allocate a new pool of at least size bytes. + * Sort it into pooltable[]. + * Mark all memory in the pool as B_FREE. + * Return the actual number of bytes reserved or 0 on error. + */ + size_t reserve(size_t size) nothrow + { + size_t npages = (size + PAGESIZE - 1) / PAGESIZE; + + // Assume reserve() is for small objects. + Pool* pool = newPool(npages, false); + + if (!pool) + return 0; + return pool.npages * PAGESIZE; + } + + /** + * Update the thresholds for when to collect the next time + */ + void updateCollectThresholds() nothrow + { + static float max(float a, float b) nothrow + { + return a >= b ? a : b; + } + + // instantly increases, slowly decreases + static float smoothDecay(float oldVal, float newVal) nothrow + { + // decay to 63.2% of newVal over 5 collections + // http://en.wikipedia.org/wiki/Low-pass_filter#Simple_infinite_impulse_response_filter + enum alpha = 1.0 / (5 + 1); + immutable decay = (newVal - oldVal) * alpha + oldVal; + return max(newVal, decay); + } + + immutable smTarget = usedSmallPages * config.heapSizeFactor; + smallCollectThreshold = smoothDecay(smallCollectThreshold, smTarget); + immutable lgTarget = usedLargePages * config.heapSizeFactor; + largeCollectThreshold = smoothDecay(largeCollectThreshold, lgTarget); + } + + /** + * Minimizes physical memory usage by returning free pools to the OS. + */ + void minimize() nothrow + { + debug(PRINTF) printf("Minimizing.\n"); + + foreach (pool; pooltable.minimize()) + { + debug(PRINTF) printFreeInfo(pool); + mappedPages -= pool.npages; + pool.Dtor(); + cstdlib.free(pool); + } + + debug(PRINTF) printf("Done minimizing.\n"); + } + + private @property bool lowMem() const nothrow + { + return isLowOnMem(mappedPages * PAGESIZE); + } + + void* alloc(size_t size, ref size_t alloc_size, uint bits) nothrow + { + return size <= 2048 ? smallAlloc(binTable[size], alloc_size, bits) + : bigAlloc(size, alloc_size, bits); + } + + void* smallAlloc(Bins bin, ref size_t alloc_size, uint bits) nothrow + { + alloc_size = binsize[bin]; + + void* p; + bool tryAlloc() nothrow + { + if (!bucket[bin]) + { + bucket[bin] = allocPage(bin); + if (!bucket[bin]) + return false; + } + p = bucket[bin]; + return true; + } + + if (!tryAlloc()) + { + if (!lowMem && (disabled || usedSmallPages < smallCollectThreshold)) + { + // disabled or threshold not reached => allocate a new pool instead of collecting + if (!newPool(1, false)) + { + // out of memory => try to free some memory + fullcollect(); + if (lowMem) minimize(); + } + } + else + { + fullcollect(); + if (lowMem) minimize(); + } + // tryAlloc will succeed if a new pool was allocated above, if it fails allocate a new pool now + if (!tryAlloc() && (!newPool(1, false) || !tryAlloc())) + // out of luck or memory + onOutOfMemoryErrorNoGC(); + } + assert(p !is null); + + // Return next item from free list + bucket[bin] = (cast(List*)p).next; + auto pool = (cast(List*)p).pool; + if (bits) + pool.setBits((p - pool.baseAddr) >> pool.shiftBy, bits); + //debug(PRINTF) printf("\tmalloc => %p\n", p); + debug (MEMSTOMP) memset(p, 0xF0, alloc_size); + return p; + } + + /** + * Allocate a chunk of memory that is larger than a page. + * Return null if out of memory. + */ + void* bigAlloc(size_t size, ref size_t alloc_size, uint bits, const TypeInfo ti = null) nothrow + { + debug(PRINTF) printf("In bigAlloc. Size: %d\n", size); + + LargeObjectPool* pool; + size_t pn; + immutable npages = (size + PAGESIZE - 1) / PAGESIZE; + if (npages == 0) + onOutOfMemoryErrorNoGC(); // size just below size_t.max requested + + bool tryAlloc() nothrow + { + foreach (p; pooltable[0 .. npools]) + { + if (!p.isLargeObject || p.freepages < npages) + continue; + auto lpool = cast(LargeObjectPool*) p; + if ((pn = lpool.allocPages(npages)) == OPFAIL) + continue; + pool = lpool; + return true; + } + return false; + } + + bool tryAllocNewPool() nothrow + { + pool = cast(LargeObjectPool*) newPool(npages, true); + if (!pool) return false; + pn = pool.allocPages(npages); + assert(pn != OPFAIL); + return true; + } + + if (!tryAlloc()) + { + if (!lowMem && (disabled || usedLargePages < largeCollectThreshold)) + { + // disabled or threshold not reached => allocate a new pool instead of collecting + if (!tryAllocNewPool()) + { + // disabled but out of memory => try to free some memory + fullcollect(); + minimize(); + } + } + else + { + fullcollect(); + minimize(); + } + // If alloc didn't yet succeed retry now that we collected/minimized + if (!pool && !tryAlloc() && !tryAllocNewPool()) + // out of luck or memory + return null; + } + assert(pool); + + debug(PRINTF) printFreeInfo(&pool.base); + pool.pagetable[pn] = B_PAGE; + if (npages > 1) + memset(&pool.pagetable[pn + 1], B_PAGEPLUS, npages - 1); + pool.updateOffsets(pn); + usedLargePages += npages; + pool.freepages -= npages; + + debug(PRINTF) printFreeInfo(&pool.base); + + auto p = pool.baseAddr + pn * PAGESIZE; + debug(PRINTF) printf("Got large alloc: %p, pt = %d, np = %d\n", p, pool.pagetable[pn], npages); + debug (MEMSTOMP) memset(p, 0xF1, size); + alloc_size = npages * PAGESIZE; + //debug(PRINTF) printf("\tp = %p\n", p); + + if (bits) + pool.setBits(pn, bits); + return p; + } + + + /** + * Allocate a new pool with at least npages in it. + * Sort it into pooltable[]. + * Return null if failed. + */ + Pool *newPool(size_t npages, bool isLargeObject) nothrow + { + //debug(PRINTF) printf("************Gcx::newPool(npages = %d)****************\n", npages); + + // Minimum of POOLSIZE + size_t minPages = (config.minPoolSize << 20) / PAGESIZE; + if (npages < minPages) + npages = minPages; + else if (npages > minPages) + { // Give us 150% of requested size, so there's room to extend + auto n = npages + (npages >> 1); + if (n < size_t.max/PAGESIZE) + npages = n; + } + + // Allocate successively larger pools up to 8 megs + if (npools) + { size_t n; + + n = config.minPoolSize + config.incPoolSize * npools; + if (n > config.maxPoolSize) + n = config.maxPoolSize; // cap pool size + n *= (1 << 20) / PAGESIZE; // convert MB to pages + if (npages < n) + npages = n; + } + + //printf("npages = %d\n", npages); + + auto pool = cast(Pool *)cstdlib.calloc(1, isLargeObject ? LargeObjectPool.sizeof : SmallObjectPool.sizeof); + if (pool) + { + pool.initialize(npages, isLargeObject); + if (!pool.baseAddr || !pooltable.insert(pool)) + { + pool.Dtor(); + cstdlib.free(pool); + return null; + } + } + + mappedPages += npages; + + if (config.profile) + { + if (mappedPages * PAGESIZE > maxPoolMemory) + maxPoolMemory = mappedPages * PAGESIZE; + } + return pool; + } + + /** + * Allocate a page of bin's. + * Returns: + * head of a single linked list of new entries + */ + List* allocPage(Bins bin) nothrow + { + //debug(PRINTF) printf("Gcx::allocPage(bin = %d)\n", bin); + for (size_t n = 0; n < npools; n++) + { + Pool* pool = pooltable[n]; + if(pool.isLargeObject) + continue; + if (List* p = (cast(SmallObjectPool*)pool).allocPage(bin)) + { + ++usedSmallPages; + return p; + } + } + return null; + } + + static struct ScanRange + { + void* pbot; + void* ptop; + } + + static struct ToScanStack + { + nothrow: + @disable this(this); + + void reset() + { + _length = 0; + os_mem_unmap(_p, _cap * ScanRange.sizeof); + _p = null; + _cap = 0; + } + + void push(ScanRange rng) + { + if (_length == _cap) grow(); + _p[_length++] = rng; + } + + ScanRange pop() + in { assert(!empty); } + do + { + return _p[--_length]; + } + + ref inout(ScanRange) opIndex(size_t idx) inout + in { assert(idx < _length); } + do + { + return _p[idx]; + } + + @property size_t length() const { return _length; } + @property bool empty() const { return !length; } + + private: + void grow() + { + pragma(inline, false); + + enum initSize = 64 * 1024; // Windows VirtualAlloc granularity + immutable ncap = _cap ? 2 * _cap : initSize / ScanRange.sizeof; + auto p = cast(ScanRange*)os_mem_map(ncap * ScanRange.sizeof); + if (p is null) onOutOfMemoryErrorNoGC(); + if (_p !is null) + { + p[0 .. _length] = _p[0 .. _length]; + os_mem_unmap(_p, _cap * ScanRange.sizeof); + } + _p = p; + _cap = ncap; + } + + size_t _length; + ScanRange* _p; + size_t _cap; + } + + ToScanStack toscan; + + /** + * Search a range of memory values and mark any pointers into the GC pool. + */ + void mark(void *pbot, void *ptop) scope nothrow + { + if (pbot >= ptop) + return; + + void **p1 = cast(void **)pbot; + void **p2 = cast(void **)ptop; + + // limit the amount of ranges added to the toscan stack + enum FANOUT_LIMIT = 32; + size_t stackPos; + ScanRange[FANOUT_LIMIT] stack = void; + + size_t pcache = 0; + + // let dmd allocate a register for this.pools + auto pools = pooltable.pools; + const highpool = pooltable.npools - 1; + const minAddr = pooltable.minAddr; + size_t memSize = pooltable.maxAddr - minAddr; + + void* base = void; + void* top = void; + + //printf("marking range: [%p..%p] (%#zx)\n", p1, p2, cast(size_t)p2 - cast(size_t)p1); + for (;;) + { + auto p = *p1; + + //if (log) debug(PRINTF) printf("\tmark %p\n", p); + if (cast(size_t)(p - minAddr) < memSize && + (cast(size_t)p & ~cast(size_t)(PAGESIZE-1)) != pcache) + { + Pool* pool = void; + size_t low = 0; + size_t high = highpool; + while (true) + { + size_t mid = (low + high) >> 1; + pool = pools[mid]; + if (p < pool.baseAddr) + high = mid - 1; + else if (p >= pool.topAddr) + low = mid + 1; + else break; + + if (low > high) + goto LnextPtr; + } + size_t offset = cast(size_t)(p - pool.baseAddr); + size_t biti = void; + size_t pn = offset / PAGESIZE; + size_t bin = pool.pagetable[pn]; // not Bins to avoid multiple size extension instructions + + //debug(PRINTF) printf("\t\tfound pool %p, base=%p, pn = %zd, bin = %d, biti = x%x\n", pool, pool.baseAddr, pn, bin, biti); + + // Adjust bit to be at start of allocated memory block + if (bin < B_PAGE) + { + // We don't care abou setting pointsToBase correctly + // because it's ignored for small object pools anyhow. + auto offsetBase = offset & notbinsize[bin]; + biti = offsetBase >> Pool.ShiftBy.Small; + //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); + + if (!pool.mark.set(biti) && !pool.noscan.test(biti)) + { + base = pool.baseAddr + offsetBase; + top = base + binsize[bin]; + goto LaddRange; + } + } + else if (bin == B_PAGE) + { + biti = offset >> Pool.ShiftBy.Large; + //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); + + pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1); + base = cast(void*)pcache; + + // For the NO_INTERIOR attribute. This tracks whether + // the pointer is an interior pointer or points to the + // base address of a block. + if(base != sentinel_sub(p) && pool.nointerior.nbits && pool.nointerior.test(biti)) + goto LnextPtr; + + if (!pool.mark.set(biti) && !pool.noscan.test(biti)) + { + top = base + pool.bPageOffsets[pn] * PAGESIZE; + goto LaddRange; + } + } + else if (bin == B_PAGEPLUS) + { + pn -= pool.bPageOffsets[pn]; + biti = pn * (PAGESIZE >> Pool.ShiftBy.Large); + + pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1); + if(pool.nointerior.nbits && pool.nointerior.test(biti)) + goto LnextPtr; + + if (!pool.mark.set(biti) && !pool.noscan.test(biti)) + { + base = pool.baseAddr + (pn * PAGESIZE); + top = base + pool.bPageOffsets[pn] * PAGESIZE; + goto LaddRange; + } + } + else + { + // Don't mark bits in B_FREE pages + assert(bin == B_FREE); + } + } + LnextPtr: + if (++p1 < p2) + continue; + + if (stackPos) + { + // pop range from local stack and recurse + auto next = &stack[--stackPos]; + p1 = cast(void**)next.pbot; + p2 = cast(void**)next.ptop; + } + else if (!toscan.empty) + { + // pop range from global stack and recurse + auto next = toscan.pop(); + p1 = cast(void**)next.pbot; + p2 = cast(void**)next.ptop; + } + else + { + // nothing more to do + break; + } + // printf(" pop [%p..%p] (%#zx)\n", p1, p2, cast(size_t)p2 - cast(size_t)p1); + pcache = 0; + continue; + + LaddRange: + if (++p1 < p2) + { + if (stackPos < stack.length) + { + stack[stackPos].pbot = base; + stack[stackPos].ptop = top; + stackPos++; + continue; + } + toscan.push(ScanRange(p1, p2)); + // reverse order for depth-first-order traversal + foreach_reverse (ref rng; stack) + toscan.push(rng); + stackPos = 0; + } + // continue with last found range + p1 = cast(void**)base; + p2 = cast(void**)top; + pcache = 0; + } + } + + // collection step 1: prepare freebits and mark bits + void prepare() nothrow + { + size_t n; + Pool* pool; + + for (n = 0; n < npools; n++) + { + pool = pooltable[n]; + pool.mark.zero(); + if(!pool.isLargeObject) pool.freebits.zero(); + } + + debug(COLLECT_PRINTF) printf("Set bits\n"); + + // Mark each free entry, so it doesn't get scanned + for (n = 0; n < B_PAGE; n++) + { + for (List *list = bucket[n]; list; list = list.next) + { + pool = list.pool; + assert(pool); + pool.freebits.set(cast(size_t)(cast(void*)list - pool.baseAddr) / 16); + } + } + + debug(COLLECT_PRINTF) printf("Marked free entries.\n"); + + for (n = 0; n < npools; n++) + { + pool = pooltable[n]; + if(!pool.isLargeObject) + { + pool.mark.copy(&pool.freebits); + } + } + } + + // collection step 2: mark roots and heap + void markAll(bool nostack) nothrow + { + if (!nostack) + { + debug(COLLECT_PRINTF) printf("\tscan stacks.\n"); + // Scan stacks and registers for each paused thread + thread_scanAll(&mark); + } + + // Scan roots[] + debug(COLLECT_PRINTF) printf("\tscan roots[]\n"); + foreach (root; roots) + { + mark(cast(void*)&root.proot, cast(void*)(&root.proot + 1)); + } + + // Scan ranges[] + debug(COLLECT_PRINTF) printf("\tscan ranges[]\n"); + //log++; + foreach (range; ranges) + { + debug(COLLECT_PRINTF) printf("\t\t%p .. %p\n", range.pbot, range.ptop); + mark(range.pbot, range.ptop); + } + //log--; + } + + // collection step 3: free all unreferenced objects + size_t sweep() nothrow + { + // Free up everything not marked + debug(COLLECT_PRINTF) printf("\tfree'ing\n"); + size_t freedLargePages; + size_t freed; + for (size_t n = 0; n < npools; n++) + { + size_t pn; + Pool* pool = pooltable[n]; + + if(pool.isLargeObject) + { + for(pn = 0; pn < pool.npages; pn++) + { + Bins bin = cast(Bins)pool.pagetable[pn]; + if(bin > B_PAGE) continue; + size_t biti = pn; + + if (!pool.mark.test(biti)) + { + void *p = pool.baseAddr + pn * PAGESIZE; + void* q = sentinel_add(p); + sentinel_Invariant(q); + + if (pool.finals.nbits && pool.finals.clear(biti)) + { + size_t size = pool.bPageOffsets[pn] * PAGESIZE - SENTINEL_EXTRA; + uint attr = pool.getBits(biti); + rt_finalizeFromGC(q, size, attr); + } + + pool.clrBits(biti, ~BlkAttr.NONE ^ BlkAttr.FINALIZE); + + debug(COLLECT_PRINTF) printf("\tcollecting big %p\n", p); + log_free(q); + pool.pagetable[pn] = B_FREE; + if(pn < pool.searchStart) pool.searchStart = pn; + freedLargePages++; + pool.freepages++; + + debug (MEMSTOMP) memset(p, 0xF3, PAGESIZE); + while (pn + 1 < pool.npages && pool.pagetable[pn + 1] == B_PAGEPLUS) + { + pn++; + pool.pagetable[pn] = B_FREE; + + // Don't need to update searchStart here because + // pn is guaranteed to be greater than last time + // we updated it. + + pool.freepages++; + freedLargePages++; + + debug (MEMSTOMP) + { p += PAGESIZE; + memset(p, 0xF3, PAGESIZE); + } + } + pool.largestFree = pool.freepages; // invalidate + } + } + } + else + { + + for (pn = 0; pn < pool.npages; pn++) + { + Bins bin = cast(Bins)pool.pagetable[pn]; + + if (bin < B_PAGE) + { + immutable size = binsize[bin]; + void *p = pool.baseAddr + pn * PAGESIZE; + void *ptop = p + PAGESIZE; + immutable base = pn * (PAGESIZE/16); + immutable bitstride = size / 16; + + bool freeBits; + PageBits toFree; + + for (size_t i; p < ptop; p += size, i += bitstride) + { + immutable biti = base + i; + + if (!pool.mark.test(biti)) + { + void* q = sentinel_add(p); + sentinel_Invariant(q); + + if (pool.finals.nbits && pool.finals.test(biti)) + rt_finalizeFromGC(q, size - SENTINEL_EXTRA, pool.getBits(biti)); + + freeBits = true; + toFree.set(i); + + debug(COLLECT_PRINTF) printf("\tcollecting %p\n", p); + log_free(sentinel_add(p)); + + debug (MEMSTOMP) memset(p, 0xF3, size); + + freed += size; + } + } + + if (freeBits) + pool.freePageBits(pn, toFree); + } + } + } + } + + assert(freedLargePages <= usedLargePages); + usedLargePages -= freedLargePages; + debug(COLLECT_PRINTF) printf("\tfree'd %u bytes, %u pages from %u pools\n", freed, freedLargePages, npools); + return freedLargePages; + } + + // collection step 4: recover pages with no live objects, rebuild free lists + size_t recover() nothrow + { + // init tail list + List**[B_PAGE] tail = void; + foreach (i, ref next; tail) + next = &bucket[i]; + + // Free complete pages, rebuild free list + debug(COLLECT_PRINTF) printf("\tfree complete pages\n"); + size_t freedSmallPages; + for (size_t n = 0; n < npools; n++) + { + size_t pn; + Pool* pool = pooltable[n]; + + if(pool.isLargeObject) + continue; + + for (pn = 0; pn < pool.npages; pn++) + { + Bins bin = cast(Bins)pool.pagetable[pn]; + size_t biti; + size_t u; + + if (bin < B_PAGE) + { + size_t size = binsize[bin]; + size_t bitstride = size / 16; + size_t bitbase = pn * (PAGESIZE / 16); + size_t bittop = bitbase + (PAGESIZE / 16); + void* p; + + biti = bitbase; + for (biti = bitbase; biti < bittop; biti += bitstride) + { + if (!pool.freebits.test(biti)) + goto Lnotfree; + } + pool.pagetable[pn] = B_FREE; + if(pn < pool.searchStart) pool.searchStart = pn; + pool.freepages++; + freedSmallPages++; + continue; + + Lnotfree: + p = pool.baseAddr + pn * PAGESIZE; + for (u = 0; u < PAGESIZE; u += size) + { + biti = bitbase + u / 16; + if (!pool.freebits.test(biti)) + continue; + auto elem = cast(List *)(p + u); + elem.pool = pool; + *tail[bin] = elem; + tail[bin] = &elem.next; + } + } + } + } + // terminate tail list + foreach (ref next; tail) + *next = null; + + assert(freedSmallPages <= usedSmallPages); + usedSmallPages -= freedSmallPages; + debug(COLLECT_PRINTF) printf("\trecovered pages = %d\n", freedSmallPages); + return freedSmallPages; + } + + /** + * Return number of full pages free'd. + */ + size_t fullcollect(bool nostack = false) nothrow + { + MonoTime start, stop, begin; + + if (config.profile) + { + begin = start = currTime; + } + + debug(COLLECT_PRINTF) printf("Gcx.fullcollect()\n"); + //printf("\tpool address range = %p .. %p\n", minAddr, maxAddr); + + { + // lock roots and ranges around suspending threads b/c they're not reentrant safe + rangesLock.lock(); + rootsLock.lock(); + scope (exit) + { + rangesLock.unlock(); + rootsLock.unlock(); + } + thread_suspendAll(); + + prepare(); + + if (config.profile) + { + stop = currTime; + prepTime += (stop - start); + start = stop; + } + + markAll(nostack); + + thread_processGCMarks(&isMarked); + thread_resumeAll(); + } + + if (config.profile) + { + stop = currTime; + markTime += (stop - start); + Duration pause = stop - begin; + if (pause > maxPauseTime) + maxPauseTime = pause; + start = stop; + } + + ConservativeGC._inFinalizer = true; + size_t freedLargePages=void; + { + scope (failure) ConservativeGC._inFinalizer = false; + freedLargePages = sweep(); + ConservativeGC._inFinalizer = false; + } + + if (config.profile) + { + stop = currTime; + sweepTime += (stop - start); + start = stop; + } + + immutable freedSmallPages = recover(); + + if (config.profile) + { + stop = currTime; + recoverTime += (stop - start); + ++numCollections; + } + + updateCollectThresholds(); + + return freedLargePages + freedSmallPages; + } + + /** + * Returns true if the addr lies within a marked block. + * + * Warning! This should only be called while the world is stopped inside + * the fullcollect function. + */ + int isMarked(void *addr) scope nothrow + { + // first, we find the Pool this block is in, then check to see if the + // mark bit is clear. + auto pool = findPool(addr); + if(pool) + { + auto offset = cast(size_t)(addr - pool.baseAddr); + auto pn = offset / PAGESIZE; + auto bins = cast(Bins)pool.pagetable[pn]; + size_t biti = void; + if(bins <= B_PAGE) + { + biti = (offset & notbinsize[bins]) >> pool.shiftBy; + } + else if(bins == B_PAGEPLUS) + { + pn -= pool.bPageOffsets[pn]; + biti = pn * (PAGESIZE >> pool.ShiftBy.Large); + } + else // bins == B_FREE + { + assert(bins == B_FREE); + return IsMarked.no; + } + return pool.mark.test(biti) ? IsMarked.yes : IsMarked.no; + } + return IsMarked.unknown; + } + + + /***** Leak Detector ******/ + + + debug (LOGGING) + { + LogArray current; + LogArray prev; + + + void log_init() + { + //debug(PRINTF) printf("+log_init()\n"); + current.reserve(1000); + prev.reserve(1000); + //debug(PRINTF) printf("-log_init()\n"); + } + + + void log_malloc(void *p, size_t size) nothrow + { + //debug(PRINTF) printf("+log_malloc(p = %p, size = %zd)\n", p, size); + Log log; + + log.p = p; + log.size = size; + log.line = ConservativeGC.line; + log.file = ConservativeGC.file; + log.parent = null; + + ConservativeGC.line = 0; + ConservativeGC.file = null; + + current.push(log); + //debug(PRINTF) printf("-log_malloc()\n"); + } + + + void log_free(void *p) nothrow @nogc + { + //debug(PRINTF) printf("+log_free(%p)\n", p); + auto i = current.find(p); + if (i == OPFAIL) + { + debug(PRINTF) printf("free'ing unallocated memory %p\n", p); + } + else + current.remove(i); + //debug(PRINTF) printf("-log_free()\n"); + } + + + void log_collect() nothrow + { + //debug(PRINTF) printf("+log_collect()\n"); + // Print everything in current that is not in prev + + debug(PRINTF) printf("New pointers this cycle: --------------------------------\n"); + size_t used = 0; + for (size_t i = 0; i < current.dim; i++) + { + auto j = prev.find(current.data[i].p); + if (j == OPFAIL) + current.data[i].print(); + else + used++; + } + + debug(PRINTF) printf("All roots this cycle: --------------------------------\n"); + for (size_t i = 0; i < current.dim; i++) + { + void* p = current.data[i].p; + if (!findPool(current.data[i].parent)) + { + auto j = prev.find(current.data[i].p); + debug(PRINTF) printf(j == OPFAIL ? "N" : " "); + current.data[i].print(); + } + } + + debug(PRINTF) printf("Used = %d-------------------------------------------------\n", used); + prev.copy(¤t); + + debug(PRINTF) printf("-log_collect()\n"); + } + + + void log_parent(void *p, void *parent) nothrow + { + //debug(PRINTF) printf("+log_parent()\n"); + auto i = current.find(p); + if (i == OPFAIL) + { + debug(PRINTF) printf("parent'ing unallocated memory %p, parent = %p\n", p, parent); + Pool *pool; + pool = findPool(p); + assert(pool); + size_t offset = cast(size_t)(p - pool.baseAddr); + size_t biti; + size_t pn = offset / PAGESIZE; + Bins bin = cast(Bins)pool.pagetable[pn]; + biti = (offset & notbinsize[bin]); + debug(PRINTF) printf("\tbin = %d, offset = x%x, biti = x%x\n", bin, offset, biti); + } + else + { + current.data[i].parent = parent; + } + //debug(PRINTF) printf("-log_parent()\n"); + } + + } + else + { + void log_init() nothrow { } + void log_malloc(void *p, size_t size) nothrow { } + void log_free(void *p) nothrow @nogc { } + void log_collect() nothrow { } + void log_parent(void *p, void *parent) nothrow { } + } +} + +/* ============================ Pool =============================== */ + +struct Pool +{ + void* baseAddr; + void* topAddr; + GCBits mark; // entries already scanned, or should not be scanned + GCBits freebits; // entries that are on the free list + GCBits finals; // entries that need finalizer run on them + GCBits structFinals;// struct entries that need a finalzier run on them + GCBits noscan; // entries that should not be scanned + GCBits appendable; // entries that are appendable + GCBits nointerior; // interior pointers should be ignored. + // Only implemented for large object pools. + size_t npages; + size_t freepages; // The number of pages not in use. + ubyte* pagetable; + + bool isLargeObject; + + enum ShiftBy + { + Small = 4, + Large = 12 + } + ShiftBy shiftBy; // shift count for the divisor used for determining bit indices. + + // This tracks how far back we have to go to find the nearest B_PAGE at + // a smaller address than a B_PAGEPLUS. To save space, we use a uint. + // This limits individual allocations to 16 terabytes, assuming a 4k + // pagesize. + uint* bPageOffsets; + + // This variable tracks a conservative estimate of where the first free + // page in this pool is, so that if a lot of pages towards the beginning + // are occupied, we can bypass them in O(1). + size_t searchStart; + size_t largestFree; // upper limit for largest free chunk in large object pool + + void initialize(size_t npages, bool isLargeObject) nothrow + { + this.isLargeObject = isLargeObject; + size_t poolsize; + + shiftBy = isLargeObject ? ShiftBy.Large : ShiftBy.Small; + + //debug(PRINTF) printf("Pool::Pool(%u)\n", npages); + poolsize = npages * PAGESIZE; + assert(poolsize >= POOLSIZE); + baseAddr = cast(byte *)os_mem_map(poolsize); + + // Some of the code depends on page alignment of memory pools + assert((cast(size_t)baseAddr & (PAGESIZE - 1)) == 0); + + if (!baseAddr) + { + //debug(PRINTF) printf("GC fail: poolsize = x%zx, errno = %d\n", poolsize, errno); + //debug(PRINTF) printf("message = '%s'\n", sys_errlist[errno]); + + npages = 0; + poolsize = 0; + } + //assert(baseAddr); + topAddr = baseAddr + poolsize; + auto nbits = cast(size_t)poolsize >> shiftBy; + + mark.alloc(nbits); + + // pagetable already keeps track of what's free for the large object + // pool. + if(!isLargeObject) + { + freebits.alloc(nbits); + } + + noscan.alloc(nbits); + appendable.alloc(nbits); + + pagetable = cast(ubyte*)cstdlib.malloc(npages); + if (!pagetable) + onOutOfMemoryErrorNoGC(); + + if(isLargeObject) + { + bPageOffsets = cast(uint*)cstdlib.malloc(npages * uint.sizeof); + if (!bPageOffsets) + onOutOfMemoryErrorNoGC(); + } + + memset(pagetable, B_FREE, npages); + + this.npages = npages; + this.freepages = npages; + this.searchStart = 0; + this.largestFree = npages; + } + + + void Dtor() nothrow + { + if (baseAddr) + { + int result; + + if (npages) + { + result = os_mem_unmap(baseAddr, npages * PAGESIZE); + assert(result == 0); + npages = 0; + } + + baseAddr = null; + topAddr = null; + } + if (pagetable) + { + cstdlib.free(pagetable); + pagetable = null; + } + + if(bPageOffsets) + cstdlib.free(bPageOffsets); + + mark.Dtor(); + if(isLargeObject) + { + nointerior.Dtor(); + } + else + { + freebits.Dtor(); + } + finals.Dtor(); + structFinals.Dtor(); + noscan.Dtor(); + appendable.Dtor(); + } + + /** + * + */ + uint getBits(size_t biti) nothrow + { + uint bits; + + if (finals.nbits && finals.test(biti)) + bits |= BlkAttr.FINALIZE; + if (structFinals.nbits && structFinals.test(biti)) + bits |= BlkAttr.STRUCTFINAL; + if (noscan.test(biti)) + bits |= BlkAttr.NO_SCAN; + if (nointerior.nbits && nointerior.test(biti)) + bits |= BlkAttr.NO_INTERIOR; + if (appendable.test(biti)) + bits |= BlkAttr.APPENDABLE; + return bits; + } + + /** + * + */ + void clrBits(size_t biti, uint mask) nothrow @nogc + { + immutable dataIndex = biti >> GCBits.BITS_SHIFT; + immutable bitOffset = biti & GCBits.BITS_MASK; + immutable keep = ~(GCBits.BITS_1 << bitOffset); + + if (mask & BlkAttr.FINALIZE && finals.nbits) + finals.data[dataIndex] &= keep; + + if (structFinals.nbits && (mask & BlkAttr.STRUCTFINAL)) + structFinals.data[dataIndex] &= keep; + + if (mask & BlkAttr.NO_SCAN) + noscan.data[dataIndex] &= keep; + if (mask & BlkAttr.APPENDABLE) + appendable.data[dataIndex] &= keep; + if (nointerior.nbits && (mask & BlkAttr.NO_INTERIOR)) + nointerior.data[dataIndex] &= keep; + } + + /** + * + */ + void setBits(size_t biti, uint mask) nothrow + { + // Calculate the mask and bit offset once and then use it to + // set all of the bits we need to set. + immutable dataIndex = biti >> GCBits.BITS_SHIFT; + immutable bitOffset = biti & GCBits.BITS_MASK; + immutable orWith = GCBits.BITS_1 << bitOffset; + + if (mask & BlkAttr.STRUCTFINAL) + { + if (!structFinals.nbits) + structFinals.alloc(mark.nbits); + structFinals.data[dataIndex] |= orWith; + } + + if (mask & BlkAttr.FINALIZE) + { + if (!finals.nbits) + finals.alloc(mark.nbits); + finals.data[dataIndex] |= orWith; + } + + if (mask & BlkAttr.NO_SCAN) + { + noscan.data[dataIndex] |= orWith; + } +// if (mask & BlkAttr.NO_MOVE) +// { +// if (!nomove.nbits) +// nomove.alloc(mark.nbits); +// nomove.data[dataIndex] |= orWith; +// } + if (mask & BlkAttr.APPENDABLE) + { + appendable.data[dataIndex] |= orWith; + } + + if (isLargeObject && (mask & BlkAttr.NO_INTERIOR)) + { + if(!nointerior.nbits) + nointerior.alloc(mark.nbits); + nointerior.data[dataIndex] |= orWith; + } + } + + void freePageBits(size_t pagenum, in ref PageBits toFree) nothrow + { + assert(!isLargeObject); + assert(!nointerior.nbits); // only for large objects + + import core.internal.traits : staticIota; + immutable beg = pagenum * (PAGESIZE / 16 / GCBits.BITS_PER_WORD); + foreach (i; staticIota!(0, PageBits.length)) + { + immutable w = toFree[i]; + if (!w) continue; + + immutable wi = beg + i; + freebits.data[wi] |= w; + noscan.data[wi] &= ~w; + appendable.data[wi] &= ~w; + } + + if (finals.nbits) + { + foreach (i; staticIota!(0, PageBits.length)) + if (toFree[i]) + finals.data[beg + i] &= ~toFree[i]; + } + + if (structFinals.nbits) + { + foreach (i; staticIota!(0, PageBits.length)) + if (toFree[i]) + structFinals.data[beg + i] &= ~toFree[i]; + } + } + + /** + * Given a pointer p in the p, return the pagenum. + */ + size_t pagenumOf(void *p) const nothrow @nogc + in + { + assert(p >= baseAddr); + assert(p < topAddr); + } + do + { + return cast(size_t)(p - baseAddr) / PAGESIZE; + } + + @property bool isFree() const pure nothrow + { + return npages == freepages; + } + + size_t slGetSize(void* p) nothrow @nogc + { + if (isLargeObject) + return (cast(LargeObjectPool*)&this).getSize(p); + else + return (cast(SmallObjectPool*)&this).getSize(p); + } + + BlkInfo slGetInfo(void* p) nothrow + { + if (isLargeObject) + return (cast(LargeObjectPool*)&this).getInfo(p); + else + return (cast(SmallObjectPool*)&this).getInfo(p); + } + + + void Invariant() const {} + + debug(INVARIANT) + invariant() + { + //mark.Invariant(); + //scan.Invariant(); + //freebits.Invariant(); + //finals.Invariant(); + //structFinals.Invariant(); + //noscan.Invariant(); + //appendable.Invariant(); + //nointerior.Invariant(); + + if (baseAddr) + { + //if (baseAddr + npages * PAGESIZE != topAddr) + //printf("baseAddr = %p, npages = %d, topAddr = %p\n", baseAddr, npages, topAddr); + assert(baseAddr + npages * PAGESIZE == topAddr); + } + + if(pagetable !is null) + { + for (size_t i = 0; i < npages; i++) + { + Bins bin = cast(Bins)pagetable[i]; + assert(bin < B_MAX); + } + } + } +} + +struct LargeObjectPool +{ + Pool base; + alias base this; + + void updateOffsets(size_t fromWhere) nothrow + { + assert(pagetable[fromWhere] == B_PAGE); + size_t pn = fromWhere + 1; + for(uint offset = 1; pn < npages; pn++, offset++) + { + if(pagetable[pn] != B_PAGEPLUS) break; + bPageOffsets[pn] = offset; + } + + // Store the size of the block in bPageOffsets[fromWhere]. + bPageOffsets[fromWhere] = cast(uint) (pn - fromWhere); + } + + /** + * Allocate n pages from Pool. + * Returns OPFAIL on failure. + */ + size_t allocPages(size_t n) nothrow + { + if(largestFree < n || searchStart + n > npages) + return OPFAIL; + + //debug(PRINTF) printf("Pool::allocPages(n = %d)\n", n); + size_t largest = 0; + if (pagetable[searchStart] == B_PAGEPLUS) + { + searchStart -= bPageOffsets[searchStart]; // jump to B_PAGE + searchStart += bPageOffsets[searchStart]; + } + while (searchStart < npages && pagetable[searchStart] == B_PAGE) + searchStart += bPageOffsets[searchStart]; + + for (size_t i = searchStart; i < npages; ) + { + assert(pagetable[i] == B_FREE); + size_t p = 1; + while (p < n && i + p < npages && pagetable[i + p] == B_FREE) + p++; + + if (p == n) + return i; + + if (p > largest) + largest = p; + + i += p; + while(i < npages && pagetable[i] == B_PAGE) + { + // we have the size information, so we skip a whole bunch of pages. + i += bPageOffsets[i]; + } + } + + // not enough free pages found, remember largest free chunk + largestFree = largest; + return OPFAIL; + } + + /** + * Free npages pages starting with pagenum. + */ + void freePages(size_t pagenum, size_t npages) nothrow @nogc + { + //memset(&pagetable[pagenum], B_FREE, npages); + if(pagenum < searchStart) + searchStart = pagenum; + + for(size_t i = pagenum; i < npages + pagenum; i++) + { + if(pagetable[i] < B_FREE) + { + freepages++; + } + + pagetable[i] = B_FREE; + } + largestFree = freepages; // invalidate + } + + /** + * Get size of pointer p in pool. + */ + size_t getSize(void *p) const nothrow @nogc + in + { + assert(p >= baseAddr); + assert(p < topAddr); + } + do + { + size_t pagenum = pagenumOf(p); + Bins bin = cast(Bins)pagetable[pagenum]; + assert(bin == B_PAGE); + return bPageOffsets[pagenum] * PAGESIZE; + } + + /** + * + */ + BlkInfo getInfo(void* p) nothrow + { + BlkInfo info; + + size_t offset = cast(size_t)(p - baseAddr); + size_t pn = offset / PAGESIZE; + Bins bin = cast(Bins)pagetable[pn]; + + if (bin == B_PAGEPLUS) + pn -= bPageOffsets[pn]; + else if (bin != B_PAGE) + return info; // no info for free pages + + info.base = baseAddr + pn * PAGESIZE; + info.size = bPageOffsets[pn] * PAGESIZE; + + info.attr = getBits(pn); + return info; + } + + void runFinalizers(in void[] segment) nothrow + { + foreach (pn; 0 .. npages) + { + Bins bin = cast(Bins)pagetable[pn]; + if (bin > B_PAGE) + continue; + size_t biti = pn; + + if (!finals.test(biti)) + continue; + + auto p = sentinel_add(baseAddr + pn * PAGESIZE); + size_t size = bPageOffsets[pn] * PAGESIZE - SENTINEL_EXTRA; + uint attr = getBits(biti); + + if(!rt_hasFinalizerInSegment(p, size, attr, segment)) + continue; + + rt_finalizeFromGC(p, size, attr); + + clrBits(biti, ~BlkAttr.NONE); + + if (pn < searchStart) + searchStart = pn; + + debug(COLLECT_PRINTF) printf("\tcollecting big %p\n", p); + //log_free(sentinel_add(p)); + + size_t n = 1; + for (; pn + n < npages; ++n) + if (pagetable[pn + n] != B_PAGEPLUS) + break; + debug (MEMSTOMP) memset(baseAddr + pn * PAGESIZE, 0xF3, n * PAGESIZE); + freePages(pn, n); + } + } +} + + +struct SmallObjectPool +{ + Pool base; + alias base this; + + /** + * Get size of pointer p in pool. + */ + size_t getSize(void *p) const nothrow @nogc + in + { + assert(p >= baseAddr); + assert(p < topAddr); + } + do + { + size_t pagenum = pagenumOf(p); + Bins bin = cast(Bins)pagetable[pagenum]; + assert(bin < B_PAGE); + return binsize[bin]; + } + + BlkInfo getInfo(void* p) nothrow + { + BlkInfo info; + size_t offset = cast(size_t)(p - baseAddr); + size_t pn = offset / PAGESIZE; + Bins bin = cast(Bins)pagetable[pn]; + + if (bin >= B_PAGE) + return info; + + info.base = cast(void*)((cast(size_t)p) & notbinsize[bin]); + info.size = binsize[bin]; + offset = info.base - baseAddr; + info.attr = getBits(cast(size_t)(offset >> ShiftBy.Small)); + + return info; + } + + void runFinalizers(in void[] segment) nothrow + { + foreach (pn; 0 .. npages) + { + Bins bin = cast(Bins)pagetable[pn]; + if (bin >= B_PAGE) + continue; + + immutable size = binsize[bin]; + auto p = baseAddr + pn * PAGESIZE; + const ptop = p + PAGESIZE; + immutable base = pn * (PAGESIZE/16); + immutable bitstride = size / 16; + + bool freeBits; + PageBits toFree; + + for (size_t i; p < ptop; p += size, i += bitstride) + { + immutable biti = base + i; + + if (!finals.test(biti)) + continue; + + auto q = sentinel_add(p); + uint attr = getBits(biti); + + if(!rt_hasFinalizerInSegment(q, size, attr, segment)) + continue; + + rt_finalizeFromGC(q, size, attr); + + freeBits = true; + toFree.set(i); + + debug(COLLECT_PRINTF) printf("\tcollecting %p\n", p); + //log_free(sentinel_add(p)); + + debug (MEMSTOMP) memset(p, 0xF3, size); + } + + if (freeBits) + freePageBits(pn, toFree); + } + } + + /** + * Allocate a page of bin's. + * Returns: + * head of a single linked list of new entries + */ + List* allocPage(Bins bin) nothrow + { + size_t pn; + for (pn = searchStart; pn < npages; pn++) + if (pagetable[pn] == B_FREE) + goto L1; + + return null; + + L1: + searchStart = pn + 1; + pagetable[pn] = cast(ubyte)bin; + freepages--; + + // Convert page to free list + size_t size = binsize[bin]; + void* p = baseAddr + pn * PAGESIZE; + void* ptop = p + PAGESIZE - size; + auto first = cast(List*) p; + + for (; p < ptop; p += size) + { + (cast(List *)p).next = cast(List *)(p + size); + (cast(List *)p).pool = &base; + } + (cast(List *)p).next = null; + (cast(List *)p).pool = &base; + return first; + } +} + +unittest // bugzilla 14467 +{ + int[] arr = new int[10]; + assert(arr.capacity); + arr = arr[$..$]; + assert(arr.capacity); +} + +unittest // bugzilla 15353 +{ + import core.memory : GC; + + static struct Foo + { + ~this() + { + GC.free(buf); // ignored in finalizer + } + + void* buf; + } + new Foo(GC.malloc(10)); + GC.collect(); +} + +unittest // bugzilla 15822 +{ + import core.memory : GC; + + __gshared ubyte[16] buf; + static struct Foo + { + ~this() + { + GC.removeRange(ptr); + GC.removeRoot(ptr); + } + + ubyte* ptr; + } + GC.addRoot(buf.ptr); + GC.addRange(buf.ptr, buf.length); + new Foo(buf.ptr); + GC.collect(); +} + +unittest // bugzilla 1180 +{ + import core.exception; + try + { + size_t x = size_t.max - 100; + byte[] big_buf = new byte[x]; + } + catch(OutOfMemoryError) + { + } +} + +/* ============================ SENTINEL =============================== */ + + +debug (SENTINEL) +{ + const size_t SENTINEL_PRE = cast(size_t) 0xF4F4F4F4F4F4F4F4UL; // 32 or 64 bits + const ubyte SENTINEL_POST = 0xF5; // 8 bits + const uint SENTINEL_EXTRA = 2 * size_t.sizeof + 1; + + + inout(size_t*) sentinel_size(inout void *p) nothrow { return &(cast(inout size_t *)p)[-2]; } + inout(size_t*) sentinel_pre(inout void *p) nothrow { return &(cast(inout size_t *)p)[-1]; } + inout(ubyte*) sentinel_post(inout void *p) nothrow { return &(cast(inout ubyte *)p)[*sentinel_size(p)]; } + + + void sentinel_init(void *p, size_t size) nothrow + { + *sentinel_size(p) = size; + *sentinel_pre(p) = SENTINEL_PRE; + *sentinel_post(p) = SENTINEL_POST; + } + + + void sentinel_Invariant(const void *p) nothrow @nogc + { + debug + { + assert(*sentinel_pre(p) == SENTINEL_PRE); + assert(*sentinel_post(p) == SENTINEL_POST); + } + else if(*sentinel_pre(p) != SENTINEL_PRE || *sentinel_post(p) != SENTINEL_POST) + onInvalidMemoryOperationError(); // also trigger in release build + } + + + void *sentinel_add(void *p) nothrow @nogc + { + return p + 2 * size_t.sizeof; + } + + + void *sentinel_sub(void *p) nothrow @nogc + { + return p - 2 * size_t.sizeof; + } +} +else +{ + const uint SENTINEL_EXTRA = 0; + + + void sentinel_init(void *p, size_t size) nothrow + { + } + + + void sentinel_Invariant(const void *p) nothrow @nogc + { + } + + + void *sentinel_add(void *p) nothrow @nogc + { + return p; + } + + + void *sentinel_sub(void *p) nothrow @nogc + { + return p; + } +} + +debug (MEMSTOMP) +unittest +{ + import core.memory; + auto p = cast(uint*)GC.malloc(uint.sizeof*3); + assert(*p == 0xF0F0F0F0); + p[2] = 0; // First two will be used for free list + GC.free(p); + assert(p[2] == 0xF2F2F2F2); +} + +debug (SENTINEL) +unittest +{ + import core.memory; + auto p = cast(ubyte*)GC.malloc(1); + assert(p[-1] == 0xF4); + assert(p[ 1] == 0xF5); +/* + p[1] = 0; + bool thrown; + try + GC.free(p); + catch (Error e) + thrown = true; + p[1] = 0xF5; + assert(thrown); +*/ +} + +unittest +{ + import core.memory; + + // https://issues.dlang.org/show_bug.cgi?id=9275 + GC.removeRoot(null); + GC.removeRoot(cast(void*)13); +} + +// improve predictability of coverage of code that is eventually not hit by other tests +unittest +{ + import core.memory; + auto p = GC.malloc(260 << 20); // new pool has 390 MB + auto q = GC.malloc(65 << 20); // next chunk (larger than 64MB to ensure the same pool is used) + auto r = GC.malloc(65 << 20); // another chunk in same pool + assert(p + (260 << 20) == q); + assert(q + (65 << 20) == r); + GC.free(q); + // should trigger "assert(bin == B_FREE);" in mark due to dangling pointer q: + GC.collect(); + // should trigger "break;" in extendNoSync: + size_t sz = GC.extend(p, 64 << 20, 66 << 20); // trigger size after p large enough (but limited) + assert(sz == 325 << 20); + GC.free(p); + GC.free(r); + r = q; // ensure q is not trashed before collection above + + p = GC.malloc(70 << 20); // from the same pool + q = GC.malloc(70 << 20); + r = GC.malloc(70 << 20); + auto s = GC.malloc(70 << 20); + auto t = GC.malloc(70 << 20); // 350 MB of 390 MB used + assert(p + (70 << 20) == q); + assert(q + (70 << 20) == r); + assert(r + (70 << 20) == s); + assert(s + (70 << 20) == t); + GC.free(r); // ensure recalculation of largestFree in nxxt allocPages + auto z = GC.malloc(75 << 20); // needs new pool + + GC.free(p); + GC.free(q); + GC.free(s); + GC.free(t); + GC.free(z); + GC.minimize(); // release huge pool +} +