diff --git a/README.md b/README.md index 1e30a37..11840e4 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ tests and further documentation are to follow when time allows. [The full API documentation is kept up-to-date on GitHub.](https://nim-works.github.io/loony/loony.html) -[The API documentation for the Ward submodule is found here.](https://nim-works.github.io/loony/loony/ward.html) +[~~The API documentation for the Ward submodule is found here.~~](https://nim-works.github.io/loony/loony/ward.html) ~~*Wards are untested and are unlikely to remain in the library*~~ #### Memory Safety & Cache Coherence @@ -114,6 +114,19 @@ committed on the push operation and read on the pop operation; this is a higher-cost primitive. You can use `unsafePush` and `unsafePop` to manipulate a `LoonyQueue` without regard to cache coherency for ultimate performance. +The LoonyQueue itself is padded across cachelines, and by default, the slots +are read and written to in a cyclic fashion over cachelines to reduce false +sharing. + +``` +Visual representation of rotating index + +| 64 bytes | 64 bytes | 64 bytes |... +| 0------- | 1------- | 2------- |... +| -63------| -64------| -65------|... +|--127-----|--128-----|--129-----|... +``` + ### Debugging Pass `--d:loonyDebug` in compilation or with a config nimscript to use debug @@ -140,8 +153,20 @@ debugNodeCounter: We recommend against changing these values unless you know what you are doing. The suggested max alignment is 16 to achieve drastically higher contention capacities. Compilation will fail if your alignment does not fit the slot count index. `-d:loonyNodeAlignment=11` - Adjust node alignment to increase/decrease contention capacity + `-d:loonySlotCount=1024` - Adjust the number of slots in each node +`-d:loonyDebug=false` - Toggle debug counters and templates, see +[debugging](#debugging). False by default. + +`-d:loonyRotate=true` - Toggle the index for the slots of +loony queue to be read over cacheline bounds in a cyclic +manner. True by default. + +> While loonyRotate is enabled, the slot count must be a +> power of 2. Error messages will indicate whether this +> is a cause of compilation failure. + ## What are Continuations? If you've somehow missed the next big thing for nim; see [CPS](https://github.com/nim-works/cps) diff --git a/loony.nim b/loony.nim index ea4e1f9..0f3da71 100644 --- a/loony.nim +++ b/loony.nim @@ -25,9 +25,16 @@ type LoonyQueue*[T] = ref LoonyQueueImpl[T] LoonyQueueImpl*[T] = object - head : Atomic[TagPtr] ## Whereby node contains the slots and idx - tail : Atomic[TagPtr] ## is the uint16 index of the slot array - currTail : Atomic[NodePtr] ## 8 bytes Current NodePtr + head {.align: 128.}: Atomic[TagPtr] ## Whereby node contains the slots and idx + tail {.align: 128.}: Atomic[TagPtr] ## is the uint16 index of the slot array + currTail {.align: 128.}: Atomic[NodePtr] ## 8 bytes Current NodePtr + # Align to 128 bytes to avoid false sharing, see: + # https://stackoverflow.com/questions/72126606/should-the-cache-padding-size-of-x86-64-be-128-bytes + # Plenty of architectural differences can impact whether + # or not 128 bytes is superior alignment to 64 bytes, but + # considering the cost that this change introduces to the + # memory consumption of the loony queue object, it is + # recommended. ## Result types for the private ## advHead and advTail functions diff --git a/loony.nimble b/loony.nimble index 12b0429..f95723f 100644 --- a/loony.nimble +++ b/loony.nimble @@ -1,4 +1,4 @@ -version = "0.3.0" +version = "0.3.1" author = "cabboose" description = "Fast mpmc queue with sympathetic memory behavior" license = "MIT" diff --git a/loony/node.nim b/loony/node.nim index 5c138a0..cd1fae3 100644 --- a/loony/node.nim +++ b/loony/node.nim @@ -69,6 +69,17 @@ else: template incEnqPathCounter*(): untyped = discard template incDeqPathCounter*(): untyped = discard +template prn*(idx: uint16): uint16 = + ## prn = 'Pro re nata' - when required + ## Provides the actual index depending on + ## if we are rotating the index or not. + when loonyRotate: + # multiply by cacheLineSize, mod by loonySlotCount + # then add idx*cacheLineSize/loonySlotCount + (idx shl lShiftBits) and (loonySlotCount - 1) or (idx shr rShiftBits) + else: + idx + template toNodePtr*(pt: uint | ptr Node): NodePtr = # Convert ptr Node into NodePtr uint cast[NodePtr](pt) @@ -105,7 +116,7 @@ proc fetchAddSlot*(t: var Node, idx: uint16, w: uint, moorder: MemoryOrder): uin ## Remembering that the pointer has 3 tail bits clear; these are ## reserved and increased atomically to indicate RESUME, READER, WRITER ## statuship. - t.slots[idx].fetchAdd(w, order = moorder) + t.slots[prn idx].fetchAdd(w, order = moorder) proc compareAndSwapNext*(t: var Node, expect: var uint, swap: uint): bool = t.next.compareExchange(expect, swap, moRelease, moRelaxed) @@ -131,7 +142,7 @@ proc allocNode*[T](pel: T): ptr Node = proc tryReclaim*(node: var Node; start: uint16) = block done: for i in start.. loonySlotCount, "Your LoonySlot count exceeds your alignment!" + doAssert loonySlotCount > 1, + "Your LoonySlot count must be greater than 1!" + when loonyRotate: + doAssert (loonySlotCount and (loonySlotCount - 1)) == 0, + fmt"Your LoonySlot count of {loonySlotCount} is not a power of 2!" & + " Either disable loonyRotate (-d:loonyRotate=false) or" & + " change the slot count." const ## Slot flag constants