Skip to content

Commit

Permalink
gcs: Pool MatchAny data allocations
Browse files Browse the repository at this point in the history
This change optimizes the memory usage of gcs.MatchAny by introducing
a sync.Pool manage allocations of uncompressed match sets.

This change improves memory utilization for wallets which must execute
MatchAny against all block cfilters during rescanning.

benchmark                            old ns/op     new ns/op     delta
BenchmarkGCSFilterBuild50000-32      4112010       3874605       -5.77%
BenchmarkGCSFilterBuild100000-32     7913372       7462646       -5.70%
BenchmarkGCSFilterMatch-32           479           486           +1.46%
BenchmarkGCSFilterMatchAny-32        1577          1425          -9.64%

benchmark                            old allocs     new allocs     delta
BenchmarkGCSFilterBuild50000-32      30             30             +0.00%
BenchmarkGCSFilterBuild100000-32     33             33             +0.00%
BenchmarkGCSFilterMatch-32           0              0              +0.00%
BenchmarkGCSFilterMatchAny-32        2              0              -100.00%

benchmark                            old bytes     new bytes     delta
BenchmarkGCSFilterBuild50000-32      1202278       1202284       +0.00%
BenchmarkGCSFilterBuild100000-32     2496604       2496586       -0.00%
BenchmarkGCSFilterMatch-32           0             0             +0.00%
BenchmarkGCSFilterMatchAny-32        176           0             -100.00%
  • Loading branch information
jrick committed Jul 12, 2018
1 parent ddd66de commit 60d7fe0
Show file tree
Hide file tree
Showing 2 changed files with 27 additions and 34 deletions.
35 changes: 27 additions & 8 deletions gcs/gcs.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"errors"
"math"
"sort"
"sync"

"github.com/aead/siphash"
"github.com/dchest/blake256"
Expand Down Expand Up @@ -39,6 +40,13 @@ var (
// SipHash keyed hash function.
const KeySize = siphash.KeySize

// uint64s implements sort.Interface for *[]uint64
type uint64s []uint64

func (s *uint64s) Len() int { return len(*s) }
func (s *uint64s) Less(i, j int) bool { return (*s)[i] < (*s)[j] }
func (s *uint64s) Swap(i, j int) { (*s)[i], (*s)[j] = (*s)[j], (*s)[i] }

// Filter describes an immutable filter that can be built from a set of data
// elements, serialized, deserialized, and queried in a thread-safe manner. The
// serialized form is compressed as a Golomb Coded Set (GCS), but does not
Expand Down Expand Up @@ -80,15 +88,15 @@ func NewFilter(P uint8, key [KeySize]byte, data [][]byte) (*Filter, error) {
}

// Allocate filter data.
values := make(uint64Slice, 0, len(data))
values := make([]uint64, 0, len(data))

// Insert the hash (modulo N*P) of each data element into a slice and
// sort the slice.
for _, d := range data {
v := siphash.Sum64(d, &key) % f.modulusNP
values = append(values, v)
}
sort.Sort(values)
sort.Sort((*uint64s)(&values))

var b bitWriter

Expand Down Expand Up @@ -257,6 +265,9 @@ func (f *Filter) Match(key [KeySize]byte, data []byte) bool {
return false
}

// matchPool pools allocations for match data.
var matchPool sync.Pool

// MatchAny checks whether any []byte value is likely (within collision
// probability) to be a member of the set represented by the filter faster than
// calling Match() for each value individually.
Expand All @@ -269,18 +280,26 @@ func (f *Filter) MatchAny(key [KeySize]byte, data [][]byte) bool {
b := newBitReader(f.filterNData[4:])

// Create an uncompressed filter of the search values.
values := make(uint64Slice, 0, len(data))
var values *[]uint64
if v := matchPool.Get(); v != nil {
values = v.(*[]uint64)
*values = (*values)[:0]
} else {
vs := make([]uint64, 0, len(data))
values = &vs
}
defer matchPool.Put(values)
for _, d := range data {
v := siphash.Sum64(d, &key) % f.modulusNP
values = append(values, v)
*values = append(*values, v)
}
sort.Sort(values)
sort.Sort((*uint64s)(values))

// Zip down the filters, comparing values until we either run out of
// values to compare in one of the filters or we reach a matching
// value.
var lastValue1, lastValue2 uint64
lastValue2 = values[0]
lastValue2 = (*values)[0]
i := 1
for lastValue1 != lastValue2 {
// Check which filter to advance to make sure we're comparing
Expand All @@ -289,8 +308,8 @@ func (f *Filter) MatchAny(key [KeySize]byte, data [][]byte) bool {
case lastValue1 > lastValue2:
// Advance filter created from search terms or return
// false if we're at the end because nothing matched.
if i < len(values) {
lastValue2 = values[i]
if i < len(*values) {
lastValue2 = (*values)[i]
i++
} else {
return false
Expand Down
26 changes: 0 additions & 26 deletions gcs/uint64slice.go

This file was deleted.

0 comments on commit 60d7fe0

Please sign in to comment.