Skip to content

Commit

Permalink
Merge pull request #40 from raulk/b32
Browse files Browse the repository at this point in the history 
Replace b58-encoded keys with b32 (no padding) + test against multiple ds. Resolves #38.
  • Loading branch information
@raulk
raulk authored Oct 11, 2018
2 parents adb7b2c + c95b4a8 commit 7cb7023
Show file tree
Hide file tree
Showing 5 changed files with 95 additions and 145 deletions.
13 changes: 7 additions & 6 deletions p2p/host/peerstore/pstoreds/addr_book.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"time"

lru "github.com/hashicorp/golang-lru"
base32 "github.com/whyrusleeping/base32"

ds "github.com/ipfs/go-datastore"
query "github.com/ipfs/go-datastore/query"
Expand All @@ -31,7 +32,7 @@ var (
)

// Peer addresses are stored under the following db key pattern:
// /peers/addr/<b58 of peer id>/<hash of maddr>
// /peers/addr/<b32 peer id no padding>/<hash of maddr>
var abBase = ds.NewKey("/peers/addrs")

var _ pstore.AddrBook = (*dsAddrBook)(nil)
Expand Down Expand Up @@ -107,7 +108,7 @@ func keysAndAddrs(p peer.ID, addrs []ma.Multiaddr) ([]ds.Key, []ma.Multiaddr, er
var (
keys = make([]ds.Key, len(addrs))
clean = make([]ma.Multiaddr, len(addrs))
parentKey = abBase.ChildString(peer.IDB58Encode(p))
parentKey = abBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p)))
i = 0
)

Expand All @@ -120,7 +121,7 @@ func keysAndAddrs(p peer.ID, addrs []ma.Multiaddr) ([]ds.Key, []ma.Multiaddr, er
if err != nil {
return nil, nil, err
}
keys[i] = parentKey.ChildString(hash.B58String())
keys[i] = parentKey.ChildString(base32.RawStdEncoding.EncodeToString(hash))
clean[i] = addr
i++
}
Expand Down Expand Up @@ -293,7 +294,7 @@ func (mgr *dsAddrBook) UpdateAddrs(p peer.ID, oldTTL time.Duration, newTTL time.

func (mgr *dsAddrBook) dbUpdateTTL(p peer.ID, oldTTL time.Duration, newTTL time.Duration) error {
var (
prefix = abBase.ChildString(peer.IDB58Encode(p))
prefix = abBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p)))
q = query.Query{Prefix: prefix.String(), KeysOnly: false}
results query.Results
err error
Expand Down Expand Up @@ -336,7 +337,7 @@ func (mgr *dsAddrBook) dbUpdateTTL(p peer.ID, oldTTL time.Duration, newTTL time.
// Addrs returns all of the non-expired addresses for a given peer.
func (mgr *dsAddrBook) Addrs(p peer.ID) []ma.Multiaddr {
var (
prefix = abBase.ChildString(peer.IDB58Encode(p))
prefix = abBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p)))
q = query.Query{Prefix: prefix.String(), KeysOnly: false, ReturnExpirations: true}
results query.Results
err error
Expand Down Expand Up @@ -411,7 +412,7 @@ func (mgr *dsAddrBook) AddrStream(ctx context.Context, p peer.ID) <-chan ma.Mult
func (mgr *dsAddrBook) ClearAddrs(p peer.ID) {
var (
err error
prefix = abBase.ChildString(peer.IDB58Encode(p))
prefix = abBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p)))
deleteFn func() error
)

Expand Down
201 changes: 72 additions & 129 deletions p2p/host/peerstore/pstoreds/ds_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,170 +2,92 @@ package pstoreds

import (
"context"
"fmt"
"io/ioutil"
"os"
"testing"
"time"

bd "github.com/dgraph-io/badger"

ds "github.com/ipfs/go-datastore"
badger "github.com/ipfs/go-ds-badger"
leveldb "github.com/ipfs/go-ds-leveldb"

pstore "github.com/libp2p/go-libp2p-peerstore"
pt "github.com/libp2p/go-libp2p-peerstore/test"
)

func BenchmarkBaselineBadgerDatastorePutEntry(b *testing.B) {
bds, closer := badgerStore(b)
defer closer()

b.ResetTimer()
for i := 0; i < b.N; i++ {
txn, _ := bds.NewTransaction(false)
type datastoreFactory func(tb testing.TB) (ds.TxnDatastore, func())

key := ds.RawKey(fmt.Sprintf("/key/%d", i))
txn.Put(key, []byte(fmt.Sprintf("/value/%d", i)))

txn.Commit()
txn.Discard()
}
var dstores = map[string]datastoreFactory{
"Badger": badgerStore,
// TODO: Enable once go-ds-leveldb supports TTL via a shim.
// "Leveldb": leveldbStore,
}

func BenchmarkBaselineBadgerDatastoreGetEntry(b *testing.B) {
bds, closer := badgerStore(b)
defer closer()

txn, _ := bds.NewTransaction(false)
keys := make([]ds.Key, 1000)
for i := 0; i < 1000; i++ {
key := ds.RawKey(fmt.Sprintf("/key/%d", i))
txn.Put(key, []byte(fmt.Sprintf("/value/%d", i)))
keys[i] = key
}
if err := txn.Commit(); err != nil {
b.Fatal(err)
}

b.ResetTimer()
for i := 0; i < b.N; i++ {
txn, _ := bds.NewTransaction(true)
if _, err := txn.Get(keys[i%1000]); err != nil {
b.Fatal(err)
}
txn.Discard()
func TestDsPeerstore(t *testing.T) {
for name, dsFactory := range dstores {
t.Run(name, func(t *testing.T) {
pt.TestPeerstore(t, peerstoreFactory(t, dsFactory, DefaultOpts()))
})
}
}

func BenchmarkBaselineBadgerDirectPutEntry(b *testing.B) {
opts := bd.DefaultOptions
func TestDsAddrBook(t *testing.T) {
for name, dsFactory := range dstores {
t.Run(name, func(t *testing.T) {
t.Run("Cacheful", func(t *testing.T) {
t.Parallel()

dataPath, err := ioutil.TempDir(os.TempDir(), "badger")
if err != nil {
b.Fatal(err)
}
opts := DefaultOpts()
opts.TTLInterval = 100 * time.Microsecond
opts.CacheSize = 1024

opts.Dir = dataPath
opts.ValueDir = dataPath
opts.SyncWrites = false
pt.TestAddrBook(t, addressBookFactory(t, dsFactory, opts))
})

db, err := bd.Open(opts)
if err != nil {
b.Fatal(err)
}
t.Run("Cacheless", func(t *testing.T) {
t.Parallel()

defer db.Close()
opts := DefaultOpts()
opts.TTLInterval = 100 * time.Microsecond
opts.CacheSize = 0

b.ResetTimer()
for i := 0; i < b.N; i++ {
txn := db.NewTransaction(true)
txn.Set([]byte(fmt.Sprintf("/key/%d", i)), []byte(fmt.Sprintf("/value/%d", i)))
txn.Commit(nil)
pt.TestAddrBook(t, addressBookFactory(t, dsFactory, opts))
})
})
}
}

func BenchmarkBaselineBadgerDirectGetEntry(b *testing.B) {
opts := bd.DefaultOptions

dataPath, err := ioutil.TempDir(os.TempDir(), "badger")
if err != nil {
b.Fatal(err)
}

opts.Dir = dataPath
opts.ValueDir = dataPath

db, err := bd.Open(opts)
if err != nil {
b.Fatal(err)
func TestDsKeyBook(t *testing.T) {
for name, dsFactory := range dstores {
t.Run(name, func(t *testing.T) {
pt.TestKeyBook(t, keyBookFactory(t, dsFactory, DefaultOpts()))
})
}

defer db.Close()

txn := db.NewTransaction(true)
for i := 0; i < 1000; i++ {
txn.Set([]byte(fmt.Sprintf("/key/%d", i)), []byte(fmt.Sprintf("/value/%d", i)))
}
txn.Commit(nil)

b.ResetTimer()
for i := 0; i < b.N; i++ {
txn := db.NewTransaction(false)
txn.Get([]byte(fmt.Sprintf("/key/%d", i%1000)))
txn.Discard()
}
}

func TestBadgerDsPeerstore(t *testing.T) {
pt.TestPeerstore(t, peerstoreFactory(t, DefaultOpts()))
}

func TestBadgerDsAddrBook(t *testing.T) {
t.Run("Cacheful", func(t *testing.T) {
t.Parallel()

opts := DefaultOpts()
opts.TTLInterval = 100 * time.Microsecond
opts.CacheSize = 1024

pt.TestAddrBook(t, addressBookFactory(t, opts))
})

t.Run("Cacheless", func(t *testing.T) {
t.Parallel()

opts := DefaultOpts()
opts.TTLInterval = 100 * time.Microsecond
opts.CacheSize = 0

pt.TestAddrBook(t, addressBookFactory(t, opts))
})
}

func TestBadgerDsKeyBook(t *testing.T) {
pt.TestKeyBook(t, keyBookFactory(t, DefaultOpts()))
}

func BenchmarkBadgerDsPeerstore(b *testing.B) {
func BenchmarkDsPeerstore(b *testing.B) {
caching := DefaultOpts()
caching.CacheSize = 1024

cacheless := DefaultOpts()
cacheless.CacheSize = 0

pt.BenchmarkPeerstore(b, peerstoreFactory(b, caching), "Caching")
pt.BenchmarkPeerstore(b, peerstoreFactory(b, cacheless), "Cacheless")
for name, dsFactory := range dstores {
b.Run(name, func(b *testing.B) {
pt.BenchmarkPeerstore(b, peerstoreFactory(b, dsFactory, caching), "Caching")
pt.BenchmarkPeerstore(b, peerstoreFactory(b, dsFactory, cacheless), "Cacheless")
})
}
}

func badgerStore(t testing.TB) (ds.TxnDatastore, func()) {
func badgerStore(tb testing.TB) (ds.TxnDatastore, func()) {
dataPath, err := ioutil.TempDir(os.TempDir(), "badger")
if err != nil {
t.Fatal(err)
tb.Fatal(err)
}
store, err := badger.NewDatastore(dataPath, nil)
if err != nil {
t.Fatal(err)
tb.Fatal(err)
}
closer := func() {
store.Close()
Expand All @@ -174,9 +96,25 @@ func badgerStore(t testing.TB) (ds.TxnDatastore, func()) {
return store, closer
}

func peerstoreFactory(tb testing.TB, opts Options) pt.PeerstoreFactory {
func leveldbStore(tb testing.TB) (ds.TxnDatastore, func()) {
dataPath, err := ioutil.TempDir(os.TempDir(), "leveldb")
if err != nil {
tb.Fatal(err)
}
store, err := leveldb.NewDatastore(dataPath, nil)
if err != nil {
tb.Fatal(err)
}
closer := func() {
store.Close()
os.RemoveAll(dataPath)
}
return store, closer
}

func peerstoreFactory(tb testing.TB, storeFactory datastoreFactory, opts Options) pt.PeerstoreFactory {
return func() (pstore.Peerstore, func()) {
store, closeFunc := badgerStore(tb)
store, closeFunc := storeFactory(tb)

ps, err := NewPeerstore(context.Background(), store, opts)
if err != nil {
Expand All @@ -187,9 +125,9 @@ func peerstoreFactory(tb testing.TB, opts Options) pt.PeerstoreFactory {
}
}

func addressBookFactory(tb testing.TB, opts Options) pt.AddrBookFactory {
func addressBookFactory(tb testing.TB, storeFactory datastoreFactory, opts Options) pt.AddrBookFactory {
return func() (pstore.AddrBook, func()) {
store, closeFunc := badgerStore(tb)
store, closeFunc := storeFactory(tb)

ab, err := NewAddrBook(context.Background(), store, opts)
if err != nil {
Expand All @@ -200,10 +138,15 @@ func addressBookFactory(tb testing.TB, opts Options) pt.AddrBookFactory {
}
}

func keyBookFactory(tb testing.TB, opts Options) pt.KeyBookFactory {
func keyBookFactory(tb testing.TB, storeFactory datastoreFactory, opts Options) pt.KeyBookFactory {
return func() (pstore.KeyBook, func()) {
store, closeFunc := badgerStore(tb)
kb, _ := NewKeyBook(context.Background(), store, opts)
store, closeFunc := storeFactory(tb)

kb, err := NewKeyBook(context.Background(), store, opts)
if err != nil {
tb.Fatal(err)
}

return kb, closeFunc
}
}
12 changes: 7 additions & 5 deletions p2p/host/peerstore/pstoreds/keybook.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"context"
"errors"

base32 "github.com/whyrusleeping/base32"

ds "github.com/ipfs/go-datastore"
query "github.com/ipfs/go-datastore/query"

Expand All @@ -13,7 +15,7 @@ import (
)

// Public and private keys are stored under the following db key pattern:
// /peers/keys/<b58 of peer id>/{pub, priv}
// /peers/keys/<b32 peer id no padding>/{pub, priv}
var (
kbBase = ds.NewKey("/peers/keys")
pubSuffix = ds.NewKey("/pub")
Expand All @@ -31,7 +33,7 @@ func NewKeyBook(_ context.Context, store ds.TxnDatastore, _ Options) (pstore.Key
}

func (kb *dsKeyBook) PubKey(p peer.ID) ic.PubKey {
key := kbBase.ChildString(peer.IDB58Encode(p)).Child(pubSuffix)
key := kbBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p))).Child(pubSuffix)

var pk ic.PubKey
if value, err := kb.ds.Get(key); err == nil {
Expand Down Expand Up @@ -68,7 +70,7 @@ func (kb *dsKeyBook) AddPubKey(p peer.ID, pk ic.PubKey) error {
return errors.New("peer ID does not match public key")
}

key := kbBase.ChildString(peer.IDB58Encode(p)).Child(pubSuffix)
key := kbBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p))).Child(pubSuffix)
val, err := pk.Bytes()
if err != nil {
log.Errorf("error while converting pubkey byte string for peer %s: %s\n", p.Pretty(), err)
Expand All @@ -82,7 +84,7 @@ func (kb *dsKeyBook) AddPubKey(p peer.ID, pk ic.PubKey) error {
}

func (kb *dsKeyBook) PrivKey(p peer.ID) ic.PrivKey {
key := kbBase.ChildString(peer.IDB58Encode(p)).Child(privSuffix)
key := kbBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p))).Child(privSuffix)
value, err := kb.ds.Get(key)
if err != nil {
log.Errorf("error while fetching privkey from datastore for peer %s: %s\n", p.Pretty(), err)
Expand All @@ -104,7 +106,7 @@ func (kb *dsKeyBook) AddPrivKey(p peer.ID, sk ic.PrivKey) error {
return errors.New("peer ID does not match private key")
}

key := kbBase.ChildString(peer.IDB58Encode(p)).Child(privSuffix)
key := kbBase.ChildString(base32.RawStdEncoding.EncodeToString([]byte(p))).Child(privSuffix)
val, err := sk.Bytes()
if err != nil {
log.Errorf("error while converting privkey byte string for peer %s: %s\n", p.Pretty(), err)
Expand Down
Loading

0 comments on commit 7cb7023

Please sign in to comment.