Skip to content

Commit

Permalink
TlSF Alloctor: use less memory for --gc:arc (#13280)
Browse files Browse the repository at this point in the history
  • Loading branch information
Araq authored Jan 28, 2020
1 parent 84e8477 commit be795bb
Show file tree
Hide file tree
Showing 3 changed files with 103 additions and 80 deletions.
179 changes: 100 additions & 79 deletions lib/system/alloc.nim
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,12 @@ type
AlignType = BiggestFloat
FreeCell {.final, pure.} = object
next: ptr FreeCell # next free cell in chunk (overlaid with refcount)
zeroField: int # 0 means cell is not used (overlaid with typ field)
# 1 means cell is manually managed pointer
# otherwise a PNimType is stored in there
when not defined(gcDestructors):
zeroField: int # 0 means cell is not used (overlaid with typ field)
# 1 means cell is manually managed pointer
# otherwise a PNimType is stored in there
else:
alignment: int

PChunk = ptr BaseChunk
PBigChunk = ptr BigChunk
Expand Down Expand Up @@ -396,8 +399,9 @@ iterator allObjects(m: var MemRegion): pointer {.inline.} =
proc iterToProc*(iter: typed, envType: typedesc; procName: untyped) {.
magic: "Plugin", compileTime.}

proc isCell(p: pointer): bool {.inline.} =
result = cast[ptr FreeCell](p).zeroField >% 1
when not defined(gcDestructors):
proc isCell(p: pointer): bool {.inline.} =
result = cast[ptr FreeCell](p).zeroField >% 1

# ------------- chunk management ----------------------------------------------
proc pageIndex(c: PChunk): int {.inline.} =
Expand Down Expand Up @@ -630,7 +634,8 @@ proc getSmallChunk(a: var MemRegion): PSmallChunk =
result = cast[PSmallChunk](res)

# -----------------------------------------------------------------------------
proc isAllocatedPtr(a: MemRegion, p: pointer): bool {.benign.}
when not defined(gcDestructors):
proc isAllocatedPtr(a: MemRegion, p: pointer): bool {.benign.}

when true:
template allocInv(a: MemRegion): bool = true
Expand Down Expand Up @@ -773,7 +778,8 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer =
inc(c.acc, size)
else:
result = c.freeList
sysAssert(c.freeList.zeroField == 0, "rawAlloc 8")
when not defined(gcDestructors):
sysAssert(c.freeList.zeroField == 0, "rawAlloc 8")
c.freeList = c.freeList.next
dec(c.free, size)
sysAssert((cast[ByteAddress](result) and (MemAlign-1)) == 0, "rawAlloc 9")
Expand Down Expand Up @@ -826,9 +832,10 @@ proc rawDealloc(a: var MemRegion, p: pointer) =
sysAssert(((cast[ByteAddress](p) and PageMask) - smallChunkOverhead()) %%
s == 0, "rawDealloc 3")
var f = cast[ptr FreeCell](p)
#echo("setting to nil: ", $cast[ByteAddress](addr(f.zeroField)))
sysAssert(f.zeroField != 0, "rawDealloc 1")
f.zeroField = 0
when not defined(gcDestructors):
#echo("setting to nil: ", $cast[ByteAddress](addr(f.zeroField)))
sysAssert(f.zeroField != 0, "rawDealloc 1")
f.zeroField = 0
f.next = c.freeList
c.freeList = f
when overwriteFree:
Expand Down Expand Up @@ -863,88 +870,102 @@ proc rawDealloc(a: var MemRegion, p: pointer) =
sysAssert(allocInv(a), "rawDealloc: end")
when logAlloc: cprintf("dealloc(pointer_%p)\n", p)

proc isAllocatedPtr(a: MemRegion, p: pointer): bool =
if isAccessible(a, p):
var c = pageAddr(p)
if not chunkUnused(c):
if isSmallChunk(c):
var c = cast[PSmallChunk](c)
var offset = (cast[ByteAddress](p) and (PageSize-1)) -%
smallChunkOverhead()
result = (c.acc >% offset) and (offset %% c.size == 0) and
(cast[ptr FreeCell](p).zeroField >% 1)
else:
var c = cast[PBigChunk](c)
result = p == addr(c.data) and cast[ptr FreeCell](p).zeroField >% 1
when not defined(gcDestructors):
proc isAllocatedPtr(a: MemRegion, p: pointer): bool =
if isAccessible(a, p):
var c = pageAddr(p)
if not chunkUnused(c):
if isSmallChunk(c):
var c = cast[PSmallChunk](c)
var offset = (cast[ByteAddress](p) and (PageSize-1)) -%
smallChunkOverhead()
result = (c.acc >% offset) and (offset %% c.size == 0) and
(cast[ptr FreeCell](p).zeroField >% 1)
else:
var c = cast[PBigChunk](c)
result = p == addr(c.data) and cast[ptr FreeCell](p).zeroField >% 1

proc prepareForInteriorPointerChecking(a: var MemRegion) {.inline.} =
a.minLargeObj = lowGauge(a.root)
a.maxLargeObj = highGauge(a.root)
proc prepareForInteriorPointerChecking(a: var MemRegion) {.inline.} =
a.minLargeObj = lowGauge(a.root)
a.maxLargeObj = highGauge(a.root)

proc interiorAllocatedPtr(a: MemRegion, p: pointer): pointer =
if isAccessible(a, p):
var c = pageAddr(p)
if not chunkUnused(c):
if isSmallChunk(c):
var c = cast[PSmallChunk](c)
var offset = (cast[ByteAddress](p) and (PageSize-1)) -%
smallChunkOverhead()
if c.acc >% offset:
sysAssert(cast[ByteAddress](addr(c.data)) +% offset ==
cast[ByteAddress](p), "offset is not what you think it is")
var d = cast[ptr FreeCell](cast[ByteAddress](addr(c.data)) +%
offset -% (offset %% c.size))
if d.zeroField >% 1:
proc interiorAllocatedPtr(a: MemRegion, p: pointer): pointer =
if isAccessible(a, p):
var c = pageAddr(p)
if not chunkUnused(c):
if isSmallChunk(c):
var c = cast[PSmallChunk](c)
var offset = (cast[ByteAddress](p) and (PageSize-1)) -%
smallChunkOverhead()
if c.acc >% offset:
sysAssert(cast[ByteAddress](addr(c.data)) +% offset ==
cast[ByteAddress](p), "offset is not what you think it is")
var d = cast[ptr FreeCell](cast[ByteAddress](addr(c.data)) +%
offset -% (offset %% c.size))
if d.zeroField >% 1:
result = d
sysAssert isAllocatedPtr(a, result), " result wrong pointer!"
else:
var c = cast[PBigChunk](c)
var d = addr(c.data)
if p >= d and cast[ptr FreeCell](d).zeroField >% 1:
result = d
sysAssert isAllocatedPtr(a, result), " result wrong pointer!"
else:
var c = cast[PBigChunk](c)
var d = addr(c.data)
if p >= d and cast[ptr FreeCell](d).zeroField >% 1:
result = d
sysAssert isAllocatedPtr(a, result), " result wrong pointer!"
else:
var q = cast[int](p)
if q >=% a.minLargeObj and q <=% a.maxLargeObj:
# this check is highly effective! Test fails for 99,96% of all checks on
# an x86-64.
var avlNode = inRange(a.root, q)
if avlNode != nil:
var k = cast[pointer](avlNode.key)
var c = cast[PBigChunk](pageAddr(k))
sysAssert(addr(c.data) == k, " k is not the same as addr(c.data)!")
if cast[ptr FreeCell](k).zeroField >% 1:
result = k
sysAssert isAllocatedPtr(a, result), " result wrong pointer!"
else:
var q = cast[int](p)
if q >=% a.minLargeObj and q <=% a.maxLargeObj:
# this check is highly effective! Test fails for 99,96% of all checks on
# an x86-64.
var avlNode = inRange(a.root, q)
if avlNode != nil:
var k = cast[pointer](avlNode.key)
var c = cast[PBigChunk](pageAddr(k))
sysAssert(addr(c.data) == k, " k is not the same as addr(c.data)!")
if cast[ptr FreeCell](k).zeroField >% 1:
result = k
sysAssert isAllocatedPtr(a, result), " result wrong pointer!"

proc ptrSize(p: pointer): int =
var x = cast[pointer](cast[ByteAddress](p) -% sizeof(FreeCell))
var c = pageAddr(p)
sysAssert(not chunkUnused(c), "ptrSize")
result = c.size -% sizeof(FreeCell)
if not isSmallChunk(c):
dec result, bigChunkOverhead()
when not defined(gcDestructors):
var x = cast[pointer](cast[ByteAddress](p) -% sizeof(FreeCell))
var c = pageAddr(p)
sysAssert(not chunkUnused(c), "ptrSize")
result = c.size -% sizeof(FreeCell)
if not isSmallChunk(c):
dec result, bigChunkOverhead()
else:
var c = pageAddr(p)
sysAssert(not chunkUnused(c), "ptrSize")
result = c.size
if not isSmallChunk(c):
dec result, bigChunkOverhead()

proc alloc(allocator: var MemRegion, size: Natural): pointer {.gcsafe.} =
result = rawAlloc(allocator, size+sizeof(FreeCell))
cast[ptr FreeCell](result).zeroField = 1 # mark it as used
sysAssert(not isAllocatedPtr(allocator, result), "alloc")
result = cast[pointer](cast[ByteAddress](result) +% sizeof(FreeCell))
track("alloc", result, size)
when not defined(gcDestructors):
result = rawAlloc(allocator, size+sizeof(FreeCell))
cast[ptr FreeCell](result).zeroField = 1 # mark it as used
sysAssert(not isAllocatedPtr(allocator, result), "alloc")
result = cast[pointer](cast[ByteAddress](result) +% sizeof(FreeCell))
track("alloc", result, size)
else:
result = rawAlloc(allocator, size)

proc alloc0(allocator: var MemRegion, size: Natural): pointer =
result = alloc(allocator, size)
zeroMem(result, size)

proc dealloc(allocator: var MemRegion, p: pointer) =
sysAssert(p != nil, "dealloc: p is nil")
var x = cast[pointer](cast[ByteAddress](p) -% sizeof(FreeCell))
sysAssert(x != nil, "dealloc: x is nil")
sysAssert(isAccessible(allocator, x), "is not accessible")
sysAssert(cast[ptr FreeCell](x).zeroField == 1, "dealloc: object header corrupted")
rawDealloc(allocator, x)
sysAssert(not isAllocatedPtr(allocator, x), "dealloc: object still accessible")
track("dealloc", p, 0)
when not defined(gcDestructors):
sysAssert(p != nil, "dealloc: p is nil")
var x = cast[pointer](cast[ByteAddress](p) -% sizeof(FreeCell))
sysAssert(x != nil, "dealloc: x is nil")
sysAssert(isAccessible(allocator, x), "is not accessible")
sysAssert(cast[ptr FreeCell](x).zeroField == 1, "dealloc: object header corrupted")
rawDealloc(allocator, x)
sysAssert(not isAllocatedPtr(allocator, x), "dealloc: object still accessible")
track("dealloc", p, 0)
else:
rawDealloc(allocator, p)

proc realloc(allocator: var MemRegion, p: pointer, newsize: Natural): pointer =
if newsize > 0:
Expand All @@ -958,7 +979,7 @@ proc realloc(allocator: var MemRegion, p: pointer, newsize: Natural): pointer =
proc realloc0(allocator: var MemRegion, p: pointer, oldsize, newsize: Natural): pointer =
result = realloc(allocator, p, newsize)
if newsize > oldsize:
zeroMem(cast[pointer](cast[int](result) + oldsize), newsize - oldsize)
zeroMem(cast[pointer](cast[uint](result) + uint(oldsize)), newsize - oldsize)

proc deallocOsPages(a: var MemRegion) =
# we free every 'ordinarily' allocated page by iterating over the page bits:
Expand Down
2 changes: 1 addition & 1 deletion lib/system/strs_v2.nim
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ proc nimAsgnStrV2(a: var NimStringV2, b: NimStringV2) {.compilerRtl.} =
a.len = b.len
a.p = b.p
else:
if isLiteral(a) or (a.p.cap and not strlitFlag) < b.len:
if isLiteral(a) or (a.p.cap and not strlitFlag) < b.len:
# we have to allocate the 'cap' here, consider
# 'let y = newStringOfCap(); var x = y'
# on the other hand... These get turned into moves now.
Expand Down
2 changes: 2 additions & 0 deletions tests/gc/gcbench.nim
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,8 @@ proc main() =
var elapsed = epochTime() - t
printDiagnostics()
echo("Completed in " & $elapsed & "s. Success!")
when declared(getMaxMem):
echo "Max memory ", formatSize getMaxMem()

when defined(GC_setMaxPause):
GC_setMaxPause 2_000
Expand Down

0 comments on commit be795bb

Please sign in to comment.