From 60615f4b3819cbe78aeb8a4bfce1d817990acd71 Mon Sep 17 00:00:00 2001
From: "Peter Johnson @insertcoffee" <wiz@wiz.co.nz>
Date: Mon, 27 Oct 2014 18:01:59 +0000
Subject: [PATCH 01/16] Update usage example, define and assign all the things

---
 README.md | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index a6666658..af20187d 100644
--- a/README.md
+++ b/README.md
@@ -15,7 +15,16 @@ since v0.
 ``` js
 // create a scuttlebutt instance and add a message to it.
 
+var keys = null;
+var HOST = '127.0.0.1';
+
+// data directories
+// mkdir /tmp/ssb1 /tmp/ssb2
+var path = '/tmp/ssb1';
+var path2 = '/tmp/ssb2';
+
 var ssb = require('secure-scuttlebutt/create')(path)
+var pull = require('pull-stream')
 
 //create a feed.
 //this represents a write access / user.
@@ -70,7 +79,7 @@ var feed2 = ssb2.createFeed()
 //follow the key we created before.
 feed.add({
   type: 'pub', address: {host: HOST, port: 1234}
-}, function () { ... })
+}, console.log.bind(console,'feed.add') )
 
 feed2.add({
   type: 'follow',

From 11dd1735d4b7caaa32e451c867f39942eea40fa7 Mon Sep 17 00:00:00 2001
From: Dominic Tarr <dominic.tarr@gmail.com>
Date: Mon, 17 Nov 2014 14:13:04 +0100
Subject: [PATCH 02/16] tests should not refer to codec directly

---
 test/feed.js                   |  4 ++--
 test/linktypes.js              |  2 +-
 test/log.js                    |  6 +++---
 test/msg-encoding.js           |  2 +-
 test/replicate-follow.js       | 28 ++++++++++++++--------------
 test/{server.js => server.js_} |  0
 test/util.js                   |  3 +--
 7 files changed, 22 insertions(+), 23 deletions(-)
 rename test/{server.js => server.js_} (100%)

diff --git a/test/feed.js b/test/feed.js
index d0424b3b..d65a1a7f 100644
--- a/test/feed.js
+++ b/test/feed.js
@@ -9,7 +9,7 @@ module.exports = function (opts) {
   tape('simple', function (t) {
 
     var db = sublevel(level('test-ssb-feed', {
-      valueEncoding: require('../codec')
+      valueEncoding: opts.codec
     }))
 
     var ssb = require('../')(db, opts)
@@ -36,7 +36,7 @@ module.exports = function (opts) {
   tape('tail', function (t) {
 
     var db = sublevel(level('test-ssb-feed2', {
-      valueEncoding: require('../codec')
+      valueEncoding: opts.codec
     }))
 
     var ssb = require('../')(db, opts)
diff --git a/test/linktypes.js b/test/linktypes.js
index 60f73208..95dae937 100644
--- a/test/linktypes.js
+++ b/test/linktypes.js
@@ -20,7 +20,7 @@ function sort (a) {
 module.exports = function (opts) {
 
   var db = sublevel(level('test-ssb-links', {
-    valueEncoding: require('../codec')
+    valueEncoding: opts.codec
   }))
 
   var create = require('../message')(opts)
diff --git a/test/log.js b/test/log.js
index a2945b7a..3a635f31 100644
--- a/test/log.js
+++ b/test/log.js
@@ -10,7 +10,7 @@ module.exports = function (opts) {
   tape('simple', function (t) {
 
     var db = sublevel(level('test-ssb-log', {
-      valueEncoding: require('../codec')
+      valueEncoding: opts.codec
     }))
 
     var ssb = require('../')(db, opts)
@@ -35,7 +35,7 @@ module.exports = function (opts) {
   tape('gt', function (t) {
 
     var db = sublevel(level('test-ssb-log2', {
-      valueEncoding: require('../codec')
+      valueEncoding: opts.codec
     }))
 
     var ssb = require('../')(db, opts)
@@ -64,7 +64,7 @@ module.exports = function (opts) {
   tape('gt 0', function (t) {
 
     var db = sublevel(level('test-ssb-log3', {
-      valueEncoding: require('../codec')
+      valueEncoding: opts.codec
     }))
 
     var ssb = require('../')(db, opts)
diff --git a/test/msg-encoding.js b/test/msg-encoding.js
index f0fa9b21..5b19350e 100644
--- a/test/msg-encoding.js
+++ b/test/msg-encoding.js
@@ -5,7 +5,7 @@ var sublevel = require('level-sublevel/bytewise')
 var pull     = require('pull-stream')
 var jhb      = require('json-human-buffer')
 
-var codec    = require('../codec')
+var codec    = require('../defaults').codec
 var hexpp    = require('hexpp')
 
 module.exports = function (opts) {
diff --git a/test/replicate-follow.js b/test/replicate-follow.js
index 7b838d05..58256726 100644
--- a/test/replicate-follow.js
+++ b/test/replicate-follow.js
@@ -2,7 +2,7 @@
 
 var pull      = require('pull-stream')
 var tape      = require('tape')
-var net       = require('net')
+//var net       = require('net')
 
 var u         = {groups: require('./group')}
 var replicate = require('../replicate')
@@ -264,17 +264,17 @@ if(!module.parent) {
     pull(a, random(), b, random(), a)
   }, 'pull-stream, random splits')
 
-  var toStream = require('pull-stream-to-stream')
-  module.exports(opts, function (a, b) {
-    var server = net.createServer(function (stream) {
-      stream.pipe(toStream(a)).pipe(stream)
-    }).listen(function () {
-      var stream = net.connect(server.address().port)
-      stream.pipe(toStream(b)).pipe(stream)
-      stream.on('close', function () {
-        server.close()
-      })
-    })
-  }, 'net')
-
+//  var toStream = require('pull-stream-to-stream')
+//  module.exports(opts, function (a, b) {
+//    var server = net.createServer(function (stream) {
+//      stream.pipe(toStream(a)).pipe(stream)
+//    }).listen(function () {
+//      var stream = net.connect(server.address().port)
+//      stream.pipe(toStream(b)).pipe(stream)
+//      stream.on('close', function () {
+//        server.close()
+//      })
+//    })
+//  }, 'net')
+//
 }
diff --git a/test/server.js b/test/server.js_
similarity index 100%
rename from test/server.js
rename to test/server.js_
diff --git a/test/util.js b/test/util.js
index b02107c7..2b3e93d6 100644
--- a/test/util.js
+++ b/test/util.js
@@ -10,7 +10,6 @@ var w         = require('./util')
 var u = w
 var replicate = require('../replicate')
 
-var codec     = require('../codec')
 var JSONB     = require('json-buffer')
 
 
@@ -28,7 +27,7 @@ module.exports = function (opts) {
 
   function createDB(name) {
     return SSB(sublevel(level(name, {
-      valueEncoding: require('../codec')
+      valueEncoding: opts.codec
     })), opts)
   }
 

From 2348452c01275ec505819868bdd0d8749173e5dc Mon Sep 17 00:00:00 2001
From: Dominic Tarr <dominic.tarr@gmail.com>
Date: Mon, 17 Nov 2014 14:18:09 +0100
Subject: [PATCH 03/16] just use jsonh encoding instead of too clever binary
 encoding

---
 defaults.js | 36 +++++++++++-------------------------
 1 file changed, 11 insertions(+), 25 deletions(-)

diff --git a/defaults.js b/defaults.js
index 87cf0265..8cb3d4aa 100644
--- a/defaults.js
+++ b/defaults.js
@@ -1,9 +1,7 @@
-
 var Blake2s = require('blake2s')
 var crypto  = require('crypto')
 var ecc     = require('eccjs')
-
-var codec   = require('./codec')
+var JSONH   = require('json-human-buffer')
 
 var curve   = ecc.curves.k256
 
@@ -31,6 +29,7 @@ module.exports = {
     generate: function () {
       return ecc.restore(curve, crypto.randomBytes(32))
     },
+
     //takes a public key and a hash and returns a signature.
     //(a signature must be a node buffer)
     sign: function (keys, hash) {
@@ -42,32 +41,19 @@ module.exports = {
     verify: function (pub, sig, hash) {
       return ecc.verify(curve, pub, sig, hash)
     },
-
-    //codec for keys. this handles serializing
-    //and deserializing keys for storage.
-    //in elliptic curves, the public key can be
-    //regenerated from the private key, so you only
-    //need to serialize the private key.
-    //in RSA, you need to remember both public and private keys.
-
-    //maybe it's a good idea to add checksums and stuff
-    //so that you can tell that this is a valid key when
-    //read off the disk?
-    codec: {
-      decode: function (buffer) {
-        return ecc.restore(curve, buffer)
-      },
-      encode: function (keys) {
-        return keys.private
-      },
-      //this makes this a valid level codec.
-      buffer: true
-    }
   },
 
   // the codec that is used to persist into leveldb.
   // this is the codec that will be passed to levelup.
   // https://github.com/rvagg/node-levelup#custom_encodings
-  codec: codec
+  codec: {
+    decode: function (string) {
+      return JSONH.parse(string)
+    },
+    encode: function (obj) {
+      return JSONH.stringify(obj, null, 2)
+    },
+    buffer: false
+  }
 }
 

From c9b136df470c93f6c85878e9970f76c911605aff Mon Sep 17 00:00:00 2001
From: Dominic Tarr <dominic.tarr@gmail.com>
Date: Mon, 17 Nov 2014 14:22:19 +0100
Subject: [PATCH 04/16] property remove direct codec dependency in tests

---
 test/history.js      | 11 ++---------
 test/msg-encoding.js |  5 +++--
 test/validation.js   |  2 +-
 3 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/test/history.js b/test/history.js
index a2c7e890..2c2e93c4 100644
--- a/test/history.js
+++ b/test/history.js
@@ -9,8 +9,7 @@ var tape      = require('tape')
 var SSB       = require('../')
 var replicate = require('../replicate')
 
-var codec     = require('../codec')
-var JSONB     = require('json-buffer')
+var JSONB     = require('json-human-buffer')
 
 var compare   = require('ltgt').compare
 
@@ -37,13 +36,7 @@ module.exports = function (opts) {
 
   function createDB(name) {
     return SSB(sublevel(level(name, {
-      valueEncoding:
-        require('../codec')
-//      {
-//        encode: JSONB.stringify,
-//        decode: JSONB.parse,
-//        buffer: false
-//      }
+      valueEncoding: opts.codec
     })), opts)
   }
 
diff --git a/test/msg-encoding.js b/test/msg-encoding.js
index 5b19350e..692c4156 100644
--- a/test/msg-encoding.js
+++ b/test/msg-encoding.js
@@ -5,11 +5,12 @@ var sublevel = require('level-sublevel/bytewise')
 var pull     = require('pull-stream')
 var jhb      = require('json-human-buffer')
 
-var codec    = require('../defaults').codec
 var hexpp    = require('hexpp')
 
 module.exports = function (opts) {
 
+  var codec = opts.codec
+
   var content = {
       "type":"post",
       "is":"text",
@@ -63,7 +64,7 @@ module.exports = function (opts) {
   tape('known error case 1', function (t) {
 
     var db = sublevel(level('test-ssb-encoding', {
-      valueEncoding: require('../codec')
+      valueEncoding: codec
     }))
     var ssb = require('../')(db, opts)
     var feed = ssb.createFeed(opts.keys.generate())
diff --git a/test/validation.js b/test/validation.js
index 760046af..fe62ab43 100644
--- a/test/validation.js
+++ b/test/validation.js
@@ -7,7 +7,7 @@ var pull     = require('pull-stream')
 module.exports = function (opts) {
 
   var db = sublevel(level('test-ssb-validate', {
-    valueEncoding: require('../codec')
+    valueEncoding: opts.codec
   }))
 
   var create = require('../message')(opts)

From e2088dde0f6d0f8da8de899f382e37c92725d6ca Mon Sep 17 00:00:00 2001
From: Dominic Tarr <dominic.tarr@gmail.com>
Date: Mon, 17 Nov 2014 14:24:09 +0100
Subject: [PATCH 05/16] remove all the varstruct stuff!

---
 codec.js     | 184 ---------------------------------------------------
 package.json |  24 +++++--
 2 files changed, 17 insertions(+), 191 deletions(-)
 delete mode 100644 codec.js

diff --git a/codec.js b/codec.js
deleted file mode 100644
index e6d9f472..00000000
--- a/codec.js
+++ /dev/null
@@ -1,184 +0,0 @@
-var svarint   = require('signed-varint')
-var varstruct = require('varstruct')
-var varmatch  = require('varstruct-match')
-var assert    = require('assert')
-var b2s       = varstruct.buffer(32)
-var signature = varstruct.buffer(64)
-var type      = varstruct.varbuf(varstruct.bound(varstruct.byte, 0, 32))
-
-var msgpack   = require('msgpack-js')
-var hexpp     = require('hexpp')
-
-var content = varstruct.varbuf(varstruct.bound(varstruct.varint, 0, 1024))
-
-var codec = exports = module.exports =
-  varmatch(varstruct.byte)
-//matches added at bottom of this file.
-
-var Message = varstruct({
-  previous  : b2s,
-  author    : b2s,
-  timestamp : varstruct.varint,
-  sequence  : varstruct.varint,
-  content   : content
-})
-
-var Ephemeral = varstruct({
-  author    : b2s,
-  sequence  : varstruct.varint,
-  ephemseq  : varstruct.varint,
-  timestamp : varstruct.varint,
-  content   : content
-})
-
-var _Signed = varstruct({
-  value: codec,
-  signature: signature
-})
-
-function clone (a) {
-  var b = {}
-  for(var k in a)
-    b[k] = a[k]
-  return b
-}
-
-var Signed = {
-  encode: function encode (_value, b, o) {
-    var value = clone(_value)
-    var sig = value.signature
-    delete value.signature
-    var r = _Signed.encode({value: value, signature: sig}, b, o)
-    encode.bytes = _Signed.encode.bytes
-    return r
-  },
-  decode: function decode (b, o) {
-    var v = _Signed.decode(b, o)
-    if(!v) {
-      decode.bytes = 0
-      return
-    }
-    v.value.signature = v.signature
-    decode.bytes = _Signed.decode.bytes
-    return v.value
-  },
-  encodingLength: function (value) {
-    value = clone(value)
-    var len = value.signature.length
-    delete value.signature
-    return codec.encodingLength(value) + len
-  }
-}
-
-function clone (v) {
-  var o = {}
-  for(var k in v)
-    o[k] = v[k]
-  return o
-}
-
-function msgpackify (codec) {
-
-  var _encode = codec.encode
-  var _decode = codec.decode
-  function encode (value, b, o) {
-    value = clone(value)
-    value.content = msgpack.encode(value.content)
-    var r = _encode(value, b, o)
-    encode.bytes = _encode.bytes
-    return r
-  }
-
-  return {
-    encode: encode,
-
-    decode: function decode (b, o) {
-      var value = _decode(b, o)
-      if(!value) {
-        decode.bytes = 0
-        return
-      }
-      value.content = msgpack.decode(value.content)
-      decode.bytes = _decode.bytes
-      return value
-    },
-    encodingLength: function (value) {
-      return encode(value).length
-    }
-  }
-  return codec
-}
-
-Message = msgpackify(Message)
-Ephemeral = msgpackify(Ephemeral)
-
-function fixed(codec, encodeAs, decodeAs) {
-  function encode (v,b,o) {
-    return codec.encode(encodeAs,b,o)
-  }
-  function decode (b,o) {
-    var v = codec.decode(b,o)
-    if(!v) {decode.bytes = 0; return}
-    assert.deepEqual(v, encodeAs)
-    decode.bytes = encodeAs.length
-    return decodeAs || v
-  }
-  var length = codec.length || codec.encodingLength(value)
-  encode.bytes = decode.bytes = length
-  return {
-    encode: encode,
-    decode: decode,
-    length: length
-  }
-}
-
-var Key = varstruct({
-  id: b2s,
-  sequence: varstruct.UInt64
-})
-
-var LatestKey = b2s
-
-function isInteger (n) {
-  return 'number' === typeof n && Math.round(n) === n
-}
-
-function isHash(b) {
-  return Buffer.isBuffer(b) && b.length == 32
-}
-
-function isString(s) {
-  return 'string' === typeof s
-}
-
-var Okay =
-  fixed(varstruct.buffer(4), new Buffer('okay'), {okay: true})
-
-var VBuffer = varstruct.varstring(varstruct.byte)
-
-codec
-  .type(0, Signed, function (t) {
-    return t.signature
-  })
-  .type(10, Message, function (t) {
-    return isHash(t.previous) && isHash(t.author)
-  })
-  .type(100, Key, function (t) {
-    return isHash(t.id) && isInteger(t.sequence)
-  })
-  .type(120, LatestKey, isHash)
-  .type(130, varstruct.varint, isInteger)
-  .type(140, Okay, function (b) {
-    return b && b.okay
-  })
-  .type(150, VBuffer, isString)
-
-exports.Message = Message
-exports.Ephemeral = Ephemeral
-exports.Signed = Signed
-
-exports.Key = Key
-
-exports.buffer = true
-exports.type = 'secure-scuttlebutt-codec'
-
diff --git a/package.json b/package.json
index 544267b0..5c1eeb2a 100644
--- a/package.json
+++ b/package.json
@@ -45,15 +45,10 @@
     "pull-stream-to-stream": "~1.3.0",
     "pull-stringify": "~1.2.2",
     "pull-switch": "~2.0.0",
-    "pull-varstruct": "~2.1.1",
     "rc": "~0.5.3",
-    "signed-varint": "~2.0.0",
     "ssb-keys": "~0.2.0",
     "ssb-msgs": "~0.1.1",
-    "stream-to-pull-stream": "~1.6.0",
-    "varint": "~3.0.1",
-    "varstruct": "~4.1.5",
-    "varstruct-match": "~2.0.1"
+    "stream-to-pull-stream": "~1.6.0"
   },
   "devDependencies": {
     "deep-equal": "^0.2.1",
@@ -70,5 +65,20 @@
     "test": "set -e; for t in test/*.js; do node $t; done"
   },
   "author": "Dominic Tarr <dominic.tarr@gmail.com> (http://dominictarr.com)",
-  "license": "MIT"
+  "license": "MIT",
+  "testling": {
+    "files": "test/defaults.js",
+    "browsers": [
+      "ie/8..latest",
+      "firefox/17..latest",
+      "firefox/nightly",
+      "chrome/22..latest",
+      "chrome/canary",
+      "opera/12..latest",
+      "opera/next",
+      "safari/5.1..latest",
+      "ipad/6.0..latest",
+      "iphone/6.0..latest"
+    ]
+  }
 }

From c692eafc331702ce80b2c2d3c07641f2be5d3716 Mon Sep 17 00:00:00 2001
From: Dominic Tarr <dominic.tarr@gmail.com>
Date: Mon, 17 Nov 2014 14:31:01 +0100
Subject: [PATCH 06/16] delete everything which has been moved into scuttlebot

---
 api.js                       |  92 ------------
 feed.js                      |  40 ++---
 index.js                     |  15 +-
 replicate.js                 |  53 -------
 rpc-server.js                |  26 ----
 server.js                    | 166 ---------------------
 test/custom-rel-replicate.js |  63 --------
 test/get-relays.js           | 104 -------------
 test/history.js              |   1 -
 test/replicate-follow.js     | 280 -----------------------------------
 test/server.js_              |  95 ------------
 test/util.js                 |   1 -
 12 files changed, 27 insertions(+), 909 deletions(-)
 delete mode 100644 api.js
 delete mode 100644 replicate.js
 delete mode 100644 rpc-server.js
 delete mode 100644 server.js
 delete mode 100644 test/custom-rel-replicate.js
 delete mode 100644 test/get-relays.js
 delete mode 100644 test/replicate-follow.js
 delete mode 100644 test/server.js_

diff --git a/api.js b/api.js
deleted file mode 100644
index cf1b99c3..00000000
--- a/api.js
+++ /dev/null
@@ -1,92 +0,0 @@
-var Serializer = require('pull-serializer')
-var JSONH = require('json-human-buffer')
-var muxrpc = require('muxrpc')
-var toPull = require('stream-to-pull-stream')
-var pull = require('pull-stream')
-
-function serialize (stream) {
-  return Serializer(stream, JSONH, {split: '\n\n'})
-}
-
-function isFunction (f) {
-  return 'function' === typeof f
-}
-
-
-var manifest = {
-  async: [
-    'add',
-    'get',
-    'getPublicKey',
-    'getLatest',
-    'whoami'
-  ],
-
-  source: [
-    'createFeedStream',
-    'createHistoryStream',
-    'createLogStream',
-    'messagesByType',
-    'messagesLinkedToMessage',
-    'messagesLinkedToFeed',
-    'messagesLinkedFromFeed',
-    'feedsLinkedToFeed',
-    'feedsLinkedFromFeed'
-  ]
-}
-
-//connect to server, and expose this api.
-exports = module.exports = function (ssb, feed) {
-
-  if(!ssb) throw new Error('ssb is required')
-  if(!feed) throw new Error('feed is required')
-
-
-  var api = {}
-  for(var key in manifest) {
-    manifest[key].forEach(function (name) {
-      api[name] = function () {
-        var args = [].slice.call(arguments)
-        var f = ssb[name].apply(ssb, args)
-        if(f)
-          return pull(f, function (read) {
-            return function (abort, cb) {
-              read(abort, function (err, data) {
-                cb(err, data)
-              })
-            }
-          })
-      }
-    })
-  }
-
-  // initialize the feed to always be with respect to
-  // a given id. or would it be better to allow access to multiple feeds?
-
-  api.add = function (data, cb) {
-    feed.add(data, cb)
-  }
-
-  api.whoami = function (_, cb) {
-    if(isFunction(_)) cb = _
-    cb(null, {id: feed.id, public: feed.keys.public})
-  }
-
-  return api
-}
-
-exports.client = function () {
-  return muxrpc(manifest, null, serialize) ()
-}
-
-exports.server = function (ssb, feed) {
-  return muxrpc(null, manifest, serialize) (exports(ssb, feed))
-}
-
-exports.peer = function (ssb, feed, _serialize) {
-  //this is terribly dangerous until we have authorization on the rpc stream
-  return muxrpc(manifest, manifest, _serialize || serialize) (exports(ssb, feed))
-      .permissions({allow: ['createHistoryStream']})
-}
-
-exports.manifest = manifest
diff --git a/feed.js b/feed.js
index 74b0d697..42ca3189 100644
--- a/feed.js
+++ b/feed.js
@@ -2,7 +2,7 @@ var cont = require('cont')
 var Message = require('./message')
 var pull = require('pull-stream')
 var cat = require('pull-cat')
-var replicate = require('./replicate')
+//var replicate = require('./replicate')
 
 function isFunction (f) {
   return 'function' === typeof f
@@ -79,24 +79,24 @@ module.exports = function (ssb, keys, opts) {
       return this
     }),
     keys: keys,
-    createReplicationStream: function (opts, cb) {
-      opts = opts || {}
-      if(!opts.latest)
-        opts.latest = function () {
-          return cat([
-            pull.values([id]),
-            pull(
-              ssb.feedsLinkedFromFeed(id, opts.$rel || 'follow'),
-              pull.map(function (link) {
-                return link.dest
-              })
-            )
-          ])
-        }
-      return replicate(
-        ssb, opts, cb || function (err) {
-          if(err) throw err
-        })
-    }
+//    createReplicationStream: function (opts, cb) {
+//      opts = opts || {}
+//      if(!opts.latest)
+//        opts.latest = function () {
+//          return cat([
+//            pull.values([id]),
+//            pull(
+//              ssb.feedsLinkedFromFeed(id, opts.$rel || 'follow'),
+//              pull.map(function (link) {
+//                return link.dest
+//              })
+//            )
+//          ])
+//        }
+//      return replicate(
+//        ssb, opts, cb || function (err) {
+//          if(err) throw err
+//        })
+//    }
   }
 }
diff --git a/index.js b/index.js
index f1de342b..758dc2a5 100644
--- a/index.js
+++ b/index.js
@@ -9,7 +9,6 @@ var Feed      = require('./feed')
 var assert    = require('assert')
 var msgpack   = require('msgpack-js')
 var ltgt      = require('ltgt')
-var replicate = require('./replicate')
 var mlib      = require('ssb-msgs')
 
 //this makes msgpack a valid level codec.
@@ -199,13 +198,13 @@ module.exports = function (db, opts) {
     )
   }
 
-  db.createReplicationStream = function (opts, cb) {
-    if(!cb) cb = opts, opts = {}
-    return replicate(db, opts, cb || function (err) {
-      if(err) throw err
-    })
-  }
-
+//  db.createReplicationStream = function (opts, cb) {
+//    if(!cb) cb = opts, opts = {}
+//    return replicate(db, opts, cb || function (err) {
+//      if(err) throw err
+//    })
+//  }
+//
   db.createFeed = function (keys) {
     if(!keys)
       keys = opts.keys.generate()
diff --git a/replicate.js b/replicate.js
deleted file mode 100644
index 799387f9..00000000
--- a/replicate.js
+++ /dev/null
@@ -1,53 +0,0 @@
-var api = require('./api')
-var manifest = api.manifest
-var goodbye = require('pull-goodbye')
-var Serializer = require('pull-serializer')
-var JSONH = require('json-human-buffer')
-var pull = require('pull-stream')
-var many = require('pull-many')
-
-function serialize (stream) {
-  return Serializer(stream, JSONH, {split: '\n\n'})
-}
-
-function id (e) {
-  return e
-}
-
-module.exports = function (ssb, opts, cb) {
-  opts = opts || {}
-  var rpc = api.peer(ssb, {}, id)
-  var progress = opts.progress || function () {}
-
-  var latestStream = opts.latest
-    ? pull(
-        opts.latest(),
-        ssb.createLatestLookupStream()
-      )
-    : ssb.latest()
-
-  var sources = many()
-  var sent = 0
-  pull(
-    latestStream,
-    pull.drain(function (upto) {
-      sources.add(rpc.createHistoryStream(upto.id, upto.sequence + 1))
-    }, function () {
-      sources.cap()
-    })
-  )
-
-  var rpcStream = rpc.createStream()
-
-  pull(
-    sources,
-    ssb.createWriteStream(function (err) {
-      rpcStream.close(function (err2) {
-        cb(err || err2)
-      })
-    })
-  )
-
-  return serialize(goodbye(rpcStream))
-
-}
diff --git a/rpc-server.js b/rpc-server.js
deleted file mode 100644
index b7321c7f..00000000
--- a/rpc-server.js
+++ /dev/null
@@ -1,26 +0,0 @@
-var toPull = require('stream-to-pull-stream')
-var net = require('net')
-var pull = require('pull-stream')
-var cat = require('pull-cat')
-var join = require('pull-join')
-var api = require('./api')
-
-
-exports = module.exports = function (ssb, feed, opts) {
-
-  return net.createServer(function (stream) {
-    stream = toPull.duplex(stream)
-    pull(stream, api.server(ssb, feed).createStream(), stream)
-  })
-  .listen(opts.rpcPort)
-
-}
-
-
-if(!module.parent) {
-  var create = require('./create')
-  var path = require('path')
-  var ssb = create(path.join(process.env.HOME, '.ssb/db'))
-  var feed = ssb.createFeed()
-  exports(ssb, feed, {port: 5656, rpcPort: 5657})
-}
diff --git a/server.js b/server.js
deleted file mode 100644
index 2d6bd4de..00000000
--- a/server.js
+++ /dev/null
@@ -1,166 +0,0 @@
-
-var toStream = require('pull-stream-to-stream')
-var toPull = require('stream-to-pull-stream')
-var pull = require('pull-stream')
-var cat = require('pull-cat')
-var join = require('pull-join')
-var api = require('./api')
-
-var rpcServer = require('./rpc-server')
-
-//search the feed for relays, connect to them and replicate.
-
-//but how is a relay stored?
-//{relay: host:port}
-//so this means we need to index keys/values/strings?
-
-function isObject (o) {
-  return o && 'object' === typeof o
-}
-
-function getRelays (ssb, id) {
-  return join(
-    pull(
-      ssb.feedsLinkedFromFeed(id, 'follow'),
-      pull.map(function (link) { return {key: link.dest}})
-    ),
-    pull(
-      ssb.messagesByType('pub'),
-      pull.map(function (msg) {
-        return {key: msg.author, value: msg.content.address}
-      })
-    ),
-    function (id, _, address) {
-      return {id: id, address: address}
-    }
-  )
-}
-
-function all(stream, cb) {
-  if (cb) return pull(stream, pull.collect(cb))
-  else return function (cb) {
-    pull(stream, pull.collect(cb))
-  }
-}
-
-function addr (opts) {
-  return opts.host + ':' + opts.port
-}
-
-
-var net = require('net')
-
-exports = module.exports = function (ssb, feed, opts) {
-
-  var connecting = false, server
-
-  var seeds = opts.seeds
-  seeds =
-    ( Array.isArray(seeds) ? seeds
-    : isObject(seeds)      ? [seeds]
-    : [])
-    .map(function (e) {
-      return {address: e, id: feed.id}
-    })
-
-  function replicated () {
-    pull(
-      ssb.latest(),
-      pull.collect(function (err, ary) {
-        if(err) return server.emit('error', err)
-        var o = {}
-        ary.forEach(function (e) {
-          o[e.id.toString('base64')] = e.sequence
-        })
-        server.emit('replicated', o)
-      })
-    )
-  }
-
-
-
-  function connect () {
-    if(connecting) return
-    connecting = true
-    if(server.closed) return
-    //TODO rewrite this to be be a realtime dataset
-    //track relay servers, and follows.
-    //keep track of how much data you have received
-    //from them and when you last replicated.
-    all(cat([
-      pull.values(seeds),
-      getRelays(ssb, feed.id)
-    ]), function (err, ary) {
-
-      //now replicate with a random relay.
-      ary = ary.filter(function (e) {
-        return e.address.port !== opts.port || e.address.host !== opts.host
-      })
-
-      var a = ary[~~(Math.random()*ary.length)]
-      if(!a || !a.address) return console.error('no one to replicate with')
-      var address = a.address
-
-      console.log(addr(opts), 'to:', addr(a.address))
-      stream = net.connect(a.address.port, a.address.host)
-
-      //when there are new messages, replicate more often.
-      //when you replicate but didn't get anything replicate slower.
-
-      stream
-        .on('error', function (err) {
-          console.log(err)
-          setTimeout(connect, 1000 + Math.random() * 3000)
-        })
-        .pipe(toStream(feed.createReplicationStream({
-            //we probably want a progress bar to show how in sync we are.
-          }, function (err) {
-            connecting = false
-            //TODO: something smarter than randomly waiting
-            setTimeout(connect, 1000 + Math.random() * 3000)
-            if(err) return server.emit('error', err)
-            replicated()
-          })))
-        .pipe(stream)
-
-    })
-
-  }
-
-  var rpc = rpcServer(ssb, feed, opts)
-
-  server = net.createServer(function (stream) {
-    stream
-      .pipe(toStream(feed.createReplicationStream({},
-        function (err) {
-          if(err) return server.emit('error', err)
-          replicated()
-        })))
-      .pipe(stream)
-  }).listen(opts.port, function () {
-    setTimeout(connect, 1000 + Math.random() * 3000)
-  })
-  .on('close', function () {
-    rpc.close()
-    server.closed = true
-  })
-
-  server.rpcServer = rpc
-
-  return server
-
-}
-
-exports.getRelays = getRelays
-
-if(!module.parent) {
-  var config = require('./config')
-  var ssbKeys = require('ssb-keys')
-
-  var path = require('path')
-  var keys = ssbKeys.loadOrCreateSync(path.join(config.path, 'secret'))
-  var create = require('./create', path.join(config.path, 'db'))
-  var ssb = create(config.path)
-  var feed = ssb.createFeed(keys)
-  exports(ssb, feed, config)
-}
diff --git a/test/custom-rel-replicate.js b/test/custom-rel-replicate.js
deleted file mode 100644
index 7f4be915..00000000
--- a/test/custom-rel-replicate.js
+++ /dev/null
@@ -1,63 +0,0 @@
-'use strict';
-
-var pull = require('pull-stream')
-var cont = require('cont')
-var tape = require('tape')
-var w = require('./util')(require('../defaults'))
-
-var ssb1 = w.createDB('simple-following1.db')
-var feed1 = ssb1.createFeed()
-
-var ssb2 = w.createDB('simple-following2.db')
-var feed2 = ssb2.createFeed()
-
-var count = 0;
-
-// Let's add some messages
-
-tape('replicate some messages', function (t) {
-
-  t.plan(8)
-
-  cont.para([
-    feed2.add({ type: 'follow', $feed: feed1.id, $rel: 'replicate' }),
-    feed1.add('msg', 'hello there! ' + count++),
-    feed1.add('msg', 'hello there! ' + count++),
-    feed1.add('msg', 'hello there! ' + count++),
-    feed1.add('msg', 'hello there! ' + count++),
-    feed1.add('msg', 'hello there! ' + count++),
-  ]) (function (err) {
-      if(err) throw err
-    // Adding a `follows` message causes feed2 to replicate messages added to feed1
-
-      var a = feed1.createReplicationStream({$rel: 'replicate'})
-      var b = feed2.createReplicationStream({$rel: 'replicate'}, function () {
-
-      pull(
-        ssb2.createLogStream(),
-        pull.drain(function (message) {
-          console.log('SSB2: ', message)
-        })
-      )
-
-      })
-
-      // pull-stream provides us with this convenient way
-      //  to connect duplex streams
-      pull(
-        a,
-        pull.through(console.log.bind(null, '>')),
-        b,
-        pull.through(console.log.bind(null, '<')),
-        a
-      )
-    })
-
-    pull(
-      ssb2.createLogStream({ live: true }),
-      pull.drain(function (message) {
-        t.ok(message)
-      })
-    )
-
-})
diff --git a/test/get-relays.js b/test/get-relays.js
deleted file mode 100644
index 6697d336..00000000
--- a/test/get-relays.js
+++ /dev/null
@@ -1,104 +0,0 @@
-'use strict';
-
-var pull    = require('pull-stream')
-var tape    = require('tape')
-var server  = require('../server')
-var cont    = require('cont')
-var compare = require('typewiselite')
-var pl      = require('pull-level')
-
-function sort (ary) {
-  return ary.sort(function (a, b) {
-    return compare(a.id, b.id) || compare(a.address, b.address)
-  })
-}
-
-function all(stream, cb) {
-  pull(stream, pull.collect(cb))
-}
-
-
-module.exports = function (opts) {
-  var w = require('./util')(opts)
-
-  tape('create an ssb retrive, relays for another feed', function (t) {
-    var ssb = w.createDB('get-relays')
-
-    var alice = ssb.createFeed()
-    var bob   = ssb.createFeed()
-
-    cont.series([
-      bob  .add('pub', {address:{host: 'localhost', port:65001}}),
-      alice.add('flw', {$feed: bob.id,   $rel: 'follow'}),
-    ]) (function (err) {
-
-        all(server.getRelays(ssb, alice.id), function (err, ary) {
-          t.deepEqual(sort(ary), sort([
-    //        {id: alice.id, address: {host: 'localhost', port: 65000}},
-            {id: bob.id, address:   {host: 'localhost', port: 65001}},
-          ]))
-          t.end()
-        })
-    })
-  })
-
-  tape('create a ssb and retrive the relays for a given feed', function (t) {
-
-    var ssb = w.createDB('get-relays2')
-
-    var alice = ssb.createFeed()
-    var bob   = ssb.createFeed()
-    var carol = ssb.createFeed()
-
-    cont.series([
-      alice.add('pub', {address:{host: 'localhost', port:65000}}),
-      bob  .add('pub', {address:{host: 'localhost', port:65001}}),
-      carol.add('pub', {address:{host: 'localhost', port:65002}}),
-      alice.add('flw', {$feed: bob.id,   $rel: 'follow'}),
-      alice.add('flw', {$feed: carol.id, $rel: 'follow'}),
-      bob  .add('flw', {$feed: alice.id, $rel: 'follow'})
-    ]) (function (err) {
-      if(err) throw err
-
-//    return pull(pl.read(ssb), pull.drain(console.log))
-//    return pull(pl.read(ssb.sublevel('idx')), pull.drain(console.log))
-
-    cont.para([
-      function (cb) {
-        all(server.getRelays(ssb, alice.id), function (err, ary) {
-          t.deepEqual(sort(ary), sort([
-    //        {id: alice.id, address: {host: 'localhost', port: 65000}},
-            {id: bob.id, address:   {host: 'localhost', port: 65001}},
-            {id: carol.id, address: {host: 'localhost', port: 65002}},
-          ]))
-          cb()
-        })
-      },
-      function (cb) {
-        all(server.getRelays(ssb, bob.id), function (err, ary) {
-          t.deepEqual(sort(ary), sort([
-            {id: alice.id, address: {host: 'localhost', port: 65000}},
-  //          {id: bob.id, address: {host: 'localhost', port:65001}},
-          ]))
-          cb()
-        })
-      },
-      function (cb) {
-        all(server.getRelays(ssb, carol.id), function (err, ary) {
-          t.deepEqual(sort(ary), sort([
-//            {id: carol.id, address: {host:'localhost', port:65002}},
-          ]))
-          cb()
-        })
-      }
-    ]) (function () {
-      t.end()
-    })
-
-    })
-  })
-}
-
-if(!module.parent) {
-  module.exports(require('../defaults'))
-}
diff --git a/test/history.js b/test/history.js
index 2c2e93c4..24f12dd8 100644
--- a/test/history.js
+++ b/test/history.js
@@ -7,7 +7,6 @@ var ecc       = require('eccjs')
 var tape      = require('tape')
 
 var SSB       = require('../')
-var replicate = require('../replicate')
 
 var JSONB     = require('json-human-buffer')
 
diff --git a/test/replicate-follow.js b/test/replicate-follow.js
deleted file mode 100644
index 58256726..00000000
--- a/test/replicate-follow.js
+++ /dev/null
@@ -1,280 +0,0 @@
-'use strict';
-
-var pull      = require('pull-stream')
-var tape      = require('tape')
-//var net       = require('net')
-
-var u         = {groups: require('./group')}
-var replicate = require('../replicate')
-var cat       = require('pull-cat')
-var random    = require('pull-randomly-split')
-
-//create a instance with a feed
-//then have another instance follow it.
-
-function rand (n) {
-  var a = []
-  while(n--)
-    a.push(Math.random())
-  return a
-}
-var z = 0
-
-module.exports = function (opts, duplexPipe, name) {
-  var s = z++
-  var w = require('./util')(opts)
-
-  //used to hash the feed output to make assertion failures easier to eyeball.
-  function hash(msg) {
-    return opts.hash(opts.codec.encode(msg)).toString('hex')
-  }
-
-  function log(name) {
-    return function (sent, recv) {
-      console.error(name, (100*sent).toPrecision(3), (100*recv).toPrecision(3))
-    }
-  }
-
-  function follow(me, other, cb) {
-    me.add('flw', {follow: {$feed: other, $rel: 'follow'}}, cb)
-  }
-
-  function createSimple(n,m) {
-    tape(name + ': simple replicate ' + JSON.stringify([n,m]), function (t) {
-      var s = ''+ z++
-
-      var ssb1 = w.createDB('sbs-replicate1_' + name + s)
-      var ssb2 = w.createDB('sbs-replicate2_' + name + s)
-
-      var cb1 = u.groups(done)
-
-      var alice = w.init2(ssb1, n, cb1())
-      var bob = w.init2(ssb2, m, cb1())
-
-      follow(alice, bob.id, cb1())
-      follow(bob, alice.id, cb1())
-
-      var ary = [], n = 1
-
-      function done (err) {
-
-        if(err) throw err
-        var cb2 = u.groups(done2)
-
-        var a = alice.createReplicationStream ({progress: log('A')}, cb2())
-        var b = bob.createReplicationStream ({progress: log('B')}, cb2())
-
-        duplexPipe(a, b)
-
-        function done2 (err) {
-          if(err) throw err
-          //now check that the databases have really been updated.
-
-          var cbs = u.groups(done3)
-
-          pull(ssb1.createFeedStream(), pull.collect(cbs()))
-          pull(ssb2.createFeedStream(), pull.collect(cbs()))
-
-          function done3 (err, ary) {
-            if(err) throw err
-            t.deepEqual(ary[0].map(hash), ary[1].map(hash))
-            t.end()
-          }
-        }
-      }
-    })
-
-  }
-
-  createSimple(1, 1)
-  createSimple(1, 2)
-  createSimple(3, 2)
-
-  function runReplication (ssb1, ssb2, alice, bob, cb) {
-
-    var cb2 = u.groups(done2)
-
-    var a = alice.createReplicationStream ({progress: log('A')}, cb2())
-    var b = bob.createReplicationStream ({progress: log('B')}, cb2())
-
-    duplexPipe(a, b)
-
-    function done2 (err) {
-      if(err) throw err
-      //now check that the databases have really been updated.
-
-      var cbs = u.groups(next)
-
-      pull(ssb1.createFeedStream(), pull.collect(cbs()))
-      pull(ssb2.createFeedStream(), pull.collect(cbs()))
-
-      function next (err, ary) {
-        cb(err, ary)
-      }
-    }
-  }
-
-  tape(name + ': replicate when already in sync', function (t) {
-    var s = ''+ z++
-
-    var ssbA = w.createDB('sbs-replicate1_' + name + s)
-    var ssbB = w.createDB('sbs-replicate2_' + name + s)
-
-    var cb1 = u.groups(next)
-
-    var alice = w.init2(ssbA, 5, cb1())
-    var bob = w.init2(ssbB, 4, cb1())
-
-    follow(bob, alice.id, cb1())
-    follow(alice, bob.id, cb1())
-
-    function next (err) {
-      if(err) throw err
-
-      runReplication(ssbA, ssbB, alice, bob, function (err, ary) {
-        t.deepEqual(ary[0].map(hash), ary[1].map(hash))
-        //*******************************************
-
-        console.log('replicate when already in sync!')
-        runReplication(ssbA, ssbB, alice, bob, function (err, ary) {
-
-          t.deepEqual(ary[0].map(hash), ary[1].map(hash))
-          console.log('replicated!!!')
-          t.end()
-
-        })
-      })
-    }
-  })
-
-  tape(name + ': replicate when already in sync', function (t) {
-
-    var s = ''+ z++
-
-    var ssbA = w.createDB('sbs-replicate1_' + name + s)
-    var ssbB = w.createDB('sbs-replicate2_' + name + s)
-
-    var cb1 = u.groups(next)
-
-    var alice = w.init2(ssbA, 5, cb1())
-    var bob = w.init2(ssbB, 4, cb1())
-
-    follow(alice, bob.id, cb1())
-    follow(bob, alice.id, cb1())
-
-    function next (err) {
-      if(err) throw err
-
-      runReplication(ssbA, ssbB, alice, bob, function (err, ary) {
-        t.deepEqual(ary[0].map(hash), ary[1].map(hash))
-        //*******************************************
-        var cb2 = u.groups(next)
-
-        w.load(ssbA, alice.keys, 3, cb2())
-        w.load(ssbA, bob.keys, 2, cb2())
-
-        function next () {
-
-          console.log('replicate after updating')
-          runReplication(ssbA, ssbB, alice, bob, function (err, ary) {
-
-            t.deepEqual(ary[0].map(hash), ary[1].map(hash))
-            console.log('replicated!!!')
-            t.end()
-          })
-        }
-      })
-    }
-  })
-
-  tape(name + ': 3-way replicate', function (t) {
-
-    var ssbA = w.createDB('sbs-3replicate1_' + name + s)
-    var ssbB = w.createDB('sbs-3replicate2_' + name + s)
-    var ssbC = w.createDB('sbs-3replicate3_' + name + s)
-
-    var cb1 = u.groups(done)
-
-    var alice = w.init2(ssbA, 10, cb1())
-    var bob   = w.init2(ssbB, 15, cb1())
-    var carol = w.init2(ssbC, 20, cb1())
-
-    follow(alice, bob.id, cb1())
-    follow(alice, carol.id, cb1())
-
-    follow(bob, alice.id, cb1())
-    follow(bob, carol.id, cb1())
-
-    follow(carol, alice.id, cb1())
-    follow(carol, bob.id, cb1())
-
-    var ary = []
-
-    function done (err) {
-      if(err) throw err
-      var cb2 = u.groups(done2)
-
-      var a = alice.createReplicationStream ({progress: log('A')}, cb2())
-      var b = bob.createReplicationStream ({progress: log('B')}, cb2())
-
-      duplexPipe(a, b)
-
-      function done2 (err) {
-        if(err) throw err
-        //now check that the databases have really been updated.
-
-        var cb3 = u.groups(done3)
-
-        var c = carol.createReplicationStream ({progress: log('C')}, cb3())
-        var b2 = bob.createReplicationStream ({progress: log('B')}, cb3())
-
-        duplexPipe(c, b2)
-
-        function done3 (err) {
-          if(err) throw err
-
-          var cbs = u.groups(done4)
-
-          pull(ssbB.createFeedStream(), pull.collect(cbs()))
-          pull(ssbC.createFeedStream(), pull.collect(cbs()))
-
-          function done4 (err, ary) {
-            if(err) throw err
-
-            t.deepEqual(ary[0].map(hash), ary[1].map(hash))
-
-            console.log('replicated!!!')
-            t.end()
-          }
-        }
-      }
-    }
-  })
-
-}
-
-if(!module.parent) {
-  var opts = require('../defaults')
-
-  module.exports(opts, function (a, b) {
-    pull(a, b, a)
-  }, 'pull-stream')
-
-  module.exports(opts, function (a, b) {
-    pull(a, random(), b, random(), a)
-  }, 'pull-stream, random splits')
-
-//  var toStream = require('pull-stream-to-stream')
-//  module.exports(opts, function (a, b) {
-//    var server = net.createServer(function (stream) {
-//      stream.pipe(toStream(a)).pipe(stream)
-//    }).listen(function () {
-//      var stream = net.connect(server.address().port)
-//      stream.pipe(toStream(b)).pipe(stream)
-//      stream.on('close', function () {
-//        server.close()
-//      })
-//    })
-//  }, 'net')
-//
-}
diff --git a/test/server.js_ b/test/server.js_
deleted file mode 100644
index 719f191d..00000000
--- a/test/server.js_
+++ /dev/null
@@ -1,95 +0,0 @@
-
-var server    = require('../server')
-var cont      = require('cont')
-var net       = require('net')
-var deepEqual = require('deep-equal')
-var tape      = require('tape')
-
-// create 3 servers
-// give them all pub servers (on localhost)
-// and get them to follow each other...
-
-module.exports = function (opts) {
-
-  tape('replicate between 3 peers', function (t) {
-
-    var u = require('./util')(opts)
-
-    var dbA = u.createDB('test-alice')
-    var alice = dbA.createFeed()
-
-    var dbB = u.createDB('test-bob')
-    var bob = dbB.createFeed()
-
-    var dbC = u.createDB('test-carol')
-    var carol = dbC.createFeed()
-
-
-    cont.para([
-      alice.add('pub', {address: {host: 'localhost', port: 45451}}),
-      bob  .add('pub', {address: {host: 'localhost', port: 45452}}),
-      carol.add('pub', {address: {host: 'localhost', port: 45453}}),
-
-      alice.add('flw', {$feed: bob.id,   $rel: 'follow'}),
-      alice.add('flw', {$feed: carol.id, $rel: 'follow'}),
-
-      bob  .add('flw', {$feed: alice.id, $rel: 'follow'}),
-      bob  .add('flw', {$feed: carol.id, $rel: 'follow'}),
-
-      carol.add('flw', {$feed: alice.id, $rel: 'follow'}),
-      carol.add('flw', {$feed: bob.id,   $rel: 'follow'})
-    ]) (function () {
-
-      //TODO: detect when everything has been replicated
-      //and end the test.
-
-      var expected = {}
-
-      expected[alice.id.toString('base64')] =
-        expected[bob  .id.toString('base64')] =
-        expected[carol.id.toString('base64')] = 4
-      function check(server, name) {
-        var closed = false
-        return server.on('replicated', function (actual) {
-          console.log(expected)
-          console.log(actual)
-          console.log('*************')
-          if(deepEqual(expected, actual) && !closed) {
-            console.log('consistent', name)
-            closed = true
-            done()
-          }
-        })
-      }
-
-      var serverA = check(server(dbA, alice, {
-        port: 45451, host: 'localhost',
-      }), 'ALICE')
-
-      var serverB = check(server(dbB, bob, {
-        port: 45452, host: 'localhost',
-        seeds: [{port: 45451, host: 'localhost'}]
-      }), 'BOB')
-
-      var serverC = check(server(dbC, carol, {
-        port: 45453, host: 'localhost',
-        seeds: [{port: 45451, host: 'localhost'}]
-      }), 'CAROL')
-
-      var n = 2
-
-      function done () {
-        if(--n) return
-        serverA.close()
-        serverB.close()
-        serverC.close()
-
-        t.end()
-      }
-
-    })
-  })
-}
-
-if(!module.parent)
-  module.exports(require('../defaults'))
diff --git a/test/util.js b/test/util.js
index 2b3e93d6..ac1a29e5 100644
--- a/test/util.js
+++ b/test/util.js
@@ -8,7 +8,6 @@ var SSB       = require('../')
 //var u         = require('../util')
 var w         = require('./util')
 var u = w
-var replicate = require('../replicate')
 
 var JSONB     = require('json-buffer')
 

From 8a2a62138047618a1f6d722462f638918615e8bd Mon Sep 17 00:00:00 2001
From: Dominic Tarr <dominic.tarr@gmail.com>
Date: Mon, 17 Nov 2014 14:35:45 +0100
Subject: [PATCH 07/16] delete out of date documentation

---
 README.md | 79 ++++---------------------------------------------------
 1 file changed, 5 insertions(+), 74 deletions(-)

diff --git a/README.md b/README.md
index af20187d..d1ed6d2d 100644
--- a/README.md
+++ b/README.md
@@ -15,16 +15,10 @@ since v0.
 ``` js
 // create a scuttlebutt instance and add a message to it.
 
-var keys = null;
-var HOST = '127.0.0.1';
-
-// data directories
-// mkdir /tmp/ssb1 /tmp/ssb2
-var path = '/tmp/ssb1';
-var path2 = '/tmp/ssb2';
-
-var ssb = require('secure-scuttlebutt/create')(path)
 var pull = require('pull-stream')
+var keys = require('ssb-keys').loadOrCreateSync(pathToSecret)
+
+var ssb = require('secure-scuttlebutt/create')('/tmp/ssb1')
 
 //create a feed.
 //this represents a write access / user.
@@ -60,36 +54,6 @@ pull(
     console.log(ary)
   })
 )
-
-// create a server for replication.
-
-var net = require('net')
-var toStream = require('pull-stream-to-stream')
-
-net.createServer(function (stream) {
-  // secure-scuttlebutt uses pull-streams so
-  // convert it into a node stream before piping.
-  stream.pipe(toStream(ssb.createReplicationStream())).pipe(stream)
-}).listen(1234)
-
-//create another database to replicate with:
-
-var ssb2 = require('secure-scuttlebutt/create')(path2)
-var feed2 = ssb2.createFeed()
-//follow the key we created before.
-feed.add({
-  type: 'pub', address: {host: HOST, port: 1234}
-}, console.log.bind(console,'feed.add') )
-
-feed2.add({
-  type: 'follow',
-  follows: {$feed: feed.id, $rel: 'follows'}
-})
-
-// replicate from the server.
-// this will pull the messages by feed1 into this database.
-var stream = net.connect(1234)
-stream.pipe(toStream(ssb2.createReplicationStream())).pipe(stream)
 ```
 
 ## Concepts
@@ -118,26 +82,8 @@ request the chain for that id, since the latest item you know about.
 
 ### Replication
 
-secure-scuttlebutt is based on [Scuttlebutt](http://www.cs.cornell.edu/home/rvr/papers/flowgossip.pdf)
-Except that there is no way for old messages to be obsoleted by new message.
-All messages are eventually replicated - secure-scuttlebutt is eventually consistent.
-
-Perhaps the simplest way to look at it is as many simultaneous
-master-slave replications. The master appends to a log,
-giving each message a monotonically increasing sequence number.
-When the slave connects to the master and requests the messages
-that came after the sequence number they received last time.
-
-Now, each node is the master of their own feed, so scuttlebutt
-replication is just like doing many simultanious master-slave replications.
-
-Since messages are signed, and replication is eventually consistent,
-it does not matter if A receives B's messages via C, they can verify
-B's messages offline (i.e, when A is out of contact with B)
-
-However, no networking is provided by this module, so that it's
-useful as a replicatable database - however, a gossip based networking
-layer is in development.
+replication has been moved into the networking layer:
+[scuttlebot](https://github.com/pfraze/scuttlebot)
 
 ### References
 
@@ -186,13 +132,6 @@ the id of the feed (which is the hash of the feeds public key)
 
 the key pair for this feed.
 
-### SecureScuttlebutt#follow (id)
-
-Mark `id`'s feed as replicated. this instance will request
-data created by `id` when replicating.
-see [createReplicationStream](#createReplicationStream)
-The id must be the hash of id's public key.
-
 ### SecureScuttlebutt#getPublicKey(id, cb)
 
 Retrive the public key for `id`, if it is in the database.
@@ -227,14 +166,6 @@ only stream messages with sequence numbers greater than `seq`.
 if `live` is true, the stream will be a
 [live mode](https://github.com/dominictarr/pull-level#example---reading)
 
-### SecureScuttlebutt#createReplicationStream()
-
-Create a duplex pull-stream that speak's secure-scuttlebutt's replication protocol.
-this will be a pull-stream so you will need to use it with 
-[pull-stream-to-stream](https://github.com/dominictarr/pull-stream-to-stream)
-
-This should be in the duplex style, when connecting as either a server or a client.
-
 ## License
 
 MIT

From 6306d2427db0c7723e70b4da2a4c53bc9909972c Mon Sep 17 00:00:00 2001
From: Dominic Tarr <dominic.tarr@gmail.com>
Date: Mon, 17 Nov 2014 16:19:36 +0100
Subject: [PATCH 08/16] use isHash, not isBuffer

---
 index.js | 21 ++++++---------------
 1 file changed, 6 insertions(+), 15 deletions(-)

diff --git a/index.js b/index.js
index 758dc2a5..d815e33b 100644
--- a/index.js
+++ b/index.js
@@ -32,14 +32,15 @@ var isBuffer = Buffer.isBuffer
 var isArray = Array.isArray
 function isObject (o) { return o && 'object' === typeof o }
 
-
 module.exports = function (db, opts) {
 
+  var isHash = opts.isHash
+
   var logDB   = db.sublevel('log')
   var feedDB  = db.sublevel('fd')
   var clockDB = db.sublevel('clk')
   var lastDB  = db.sublevel('lst')
-  var indexDB = db.sublevel('idx', {valueEncoding: msgpack})
+  var indexDB = db.sublevel('idx')
   var appsDB  = db.sublevel('app')
 
   function get (db, key) {
@@ -48,14 +49,13 @@ module.exports = function (db, opts) {
 
   db.opts = opts
 
-  var isHash = opts.isHash
-
   var validation = require('./validation')(db, opts)
 
   db.pre(function (op, add, _batch) {
     var msg = op.value
     var id = op.key
     // index by sequence number
+
     add({
       key: [msg.author, msg.sequence], value: id,
       type: 'put', prefix: clockDB
@@ -89,7 +89,6 @@ module.exports = function (db, opts) {
     })
 
     mlib.indexLinks(msg.content, function (link) {
-
       if(isHash(link.$feed)) {
         add({
           key: ['feed', msg.author, link.$rel, link.$feed, msg.sequence, id],
@@ -106,7 +105,6 @@ module.exports = function (db, opts) {
       if(isHash(link.$msg)) {
         // do not need forward index here, because
         // it's cheap to just read the message.
-
         add({
           key: ['_msg', link.$msg, link.$rel, id], value: link,
           type: 'put', prefix: indexDB
@@ -170,7 +168,7 @@ module.exports = function (db, opts) {
   }
 
   db.createHistoryStream = function (id, seq, live) {
-    if(!Buffer.isBuffer(id)) {
+    if(!isHash(id)) {
       live = !!id.live
       seq = id.sequence || id.seq || 0
       id = id.id
@@ -198,13 +196,6 @@ module.exports = function (db, opts) {
     )
   }
 
-//  db.createReplicationStream = function (opts, cb) {
-//    if(!cb) cb = opts, opts = {}
-//    return replicate(db, opts, cb || function (err) {
-//      if(err) throw err
-//    })
-//  }
-//
   db.createFeed = function (keys) {
     if(!keys)
       keys = opts.keys.generate()
@@ -276,7 +267,7 @@ module.exports = function (db, opts) {
     return function (opts, rel) {
       if(!opts) throw new Error('must have opts')
       //legacy interface.
-      if(Buffer.isBuffer(opts))
+      if(isHash(opts))
         return fn(opts, rel)
 
       return fn(opts.id, opts.rel)

From be18e46e7419207348115f39724e06bc04023ad3 Mon Sep 17 00:00:00 2001
From: Dominic Tarr <dominic.tarr@gmail.com>
Date: Mon, 17 Nov 2014 16:20:09 +0100
Subject: [PATCH 09/16] put the chosen hash in the message

---
 message.js | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/message.js b/message.js
index 774b3e30..39f711b5 100644
--- a/message.js
+++ b/message.js
@@ -9,8 +9,8 @@ function isString (s) {
 
 module.exports = function (opts) {
 
-  var zeros = opts.hash(new Buffer(0))
-  zeros.fill(0)
+//  var zeros = opts.hash(new Buffer(0))
+//  zeros.fill(0)
 
   function sign (msg, keys) {
 
@@ -30,10 +30,11 @@ module.exports = function (opts) {
     //noise end
 
     return sign({
-      previous: prev ? opts.hash(opts.codec.encode(prev)) : zeros,
+      previous: prev ? opts.hash(opts.codec.encode(prev)) : null,
       author: opts.hash(keys.public),
       sequence: prev ? prev.sequence + 1 : 1,
       timestamp: Date.now(),
+      hash: 'blake2s',
       content: content,
     }, keys)
   }

From fff071386835afb3191a9c688ca422976800b683 Mon Sep 17 00:00:00 2001
From: Dominic Tarr <dominic.tarr@gmail.com>
Date: Mon, 17 Nov 2014 16:21:38 +0100
Subject: [PATCH 10/16] tag the hash with the current algorithm

---
 defaults.js | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/defaults.js b/defaults.js
index 8cb3d4aa..f963f89e 100644
--- a/defaults.js
+++ b/defaults.js
@@ -11,16 +11,27 @@ var curve   = ecc.curves.k256
 // be handy for forks to be able to use different
 // crypto or encodings etc.
 
+function isString(s) {
+  return 'string' === typeof s
+}
+function isHash (data) {
+  return isString(data) && /^[A-Za-z0-9\/+]{43}=\.blake2s$/.test(data)
+  //return Buffer.isBuffer(data) && data.length == 32
+}
+
+function toBuffer(hash) {
+  if(!isHash(hash)) throw new Error('sign expects a hash')
+  return new Buffer(hash.substring(0, 44), 'base64')
+}
+
 module.exports = {
 
   //this must return a buffer digest.
   hash: function (data, enc) {
-    return new Blake2s().update(data, enc).digest()
+    return new Blake2s().update(data, enc).digest('base64') + '.blake2s'
   },
 
-  isHash: function (data) {
-    return Buffer.isBuffer(data) && data.length == 32
-  },
+  isHash: isHash,
 
   keys: {
     //this should return a key pair:
@@ -33,13 +44,13 @@ module.exports = {
     //takes a public key and a hash and returns a signature.
     //(a signature must be a node buffer)
     sign: function (keys, hash) {
-      return ecc.sign(curve, keys, hash)
+      return ecc.sign(curve, keys, toBuffer(hash))
     },
 
     //takes a public key, signature, and a hash
     //and returns true if the signature was valid.
     verify: function (pub, sig, hash) {
-      return ecc.verify(curve, pub, sig, hash)
+      return ecc.verify(curve, pub, sig, toBuffer(hash))
     },
   },
 

From 96adb467727f7a2abadec6aac2df44de9ed84555 Mon Sep 17 00:00:00 2001
From: Dominic Tarr <dominic.tarr@gmail.com>
Date: Mon, 17 Nov 2014 16:22:33 +0100
Subject: [PATCH 11/16] allow previous on init to be nullish

---
 validation.js | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/validation.js b/validation.js
index ddf6f2c3..3ee0c8ea 100644
--- a/validation.js
+++ b/validation.js
@@ -28,8 +28,9 @@ module.exports = function (ssb, opts) {
 
   var lastDB = ssb.sublevel('lst')
   var hash = opts.hash
-  var zeros = opts.hash(new Buffer(0))
-      zeros.fill(0)
+  var zeros = undefined
+      //opts.hash(new Buffer(0))
+      //zeros.fill(0)
 
   var verify = opts.keys.verify
   var encode = opts.codec.encode
@@ -65,8 +66,8 @@ module.exports = function (ssb, opts) {
       }
     }
     else {
-      if(!deepEqual(msg.previous, zeros)
-        && msg.sequence === 1 && msg.timestamp > 0) {
+      if(!(msg.previous == null
+        && msg.sequence === 1 && msg.timestamp > 0)) {
 
           validateSync.reason = 'expected initial message'
 
@@ -91,6 +92,7 @@ module.exports = function (ssb, opts) {
 
       return false
     }
+    validateSync.reason = ''
     return true
   }
 

From 65ace418e4c6567b5e02fea521217d53f239e374 Mon Sep 17 00:00:00 2001
From: Dominic Tarr <dominic.tarr@gmail.com>
Date: Tue, 18 Nov 2014 12:42:02 +0100
Subject: [PATCH 12/16] tidy

---
 test/history.js | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/test/history.js b/test/history.js
index 24f12dd8..a4d431fd 100644
--- a/test/history.js
+++ b/test/history.js
@@ -8,8 +8,6 @@ var tape      = require('tape')
 
 var SSB       = require('../')
 
-var JSONB     = require('json-human-buffer')
-
 var compare   = require('ltgt').compare
 
 //create a instance with a feed
@@ -93,7 +91,6 @@ module.exports = function (opts) {
     keys2 = init(ssb, 4, function (err) {
       pull(ssb.latest(), pull.collect(function (err, ary) {
         if(err) throw err
-        console.log(ary)
         t.deepEqual(sort(ary), sort([
           {id: id, sequence: 8},
           {id: id2, sequence: 5}

From a4d11e9e9f4b56b5318d82d1038aa877b86e6483 Mon Sep 17 00:00:00 2001
From: Dominic Tarr <dominic.tarr@gmail.com>
Date: Tue, 18 Nov 2014 13:48:44 +0100
Subject: [PATCH 13/16] note about createFeedStream

---
 index.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/index.js b/index.js
index d815e33b..2a440b12 100644
--- a/index.js
+++ b/index.js
@@ -146,6 +146,7 @@ module.exports = function (db, opts) {
     })
   }
 
+  //TODO: eventually, this should filter out authors you do not follow.
   db.createFeedStream = function (opts) {
     opts = opts || {}
     opts.keys = false

From 468cb0b47f3439a30d94f0e556c6d396c6667150 Mon Sep 17 00:00:00 2001
From: Dominic Tarr <dominic.tarr@gmail.com>
Date: Tue, 18 Nov 2014 13:49:37 +0100
Subject: [PATCH 14/16] use explain-error to make test/validation easier to
 debug

---
 test/validation.js | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/test/validation.js b/test/validation.js
index fe62ab43..b2c7556e 100644
--- a/test/validation.js
+++ b/test/validation.js
@@ -3,6 +3,7 @@ var tape     = require('tape')
 var level    = require('level-test')()
 var sublevel = require('level-sublevel/bytewise')
 var pull     = require('pull-stream')
+var explain  = require('explain-error')
 
 module.exports = function (opts) {
 
@@ -47,21 +48,21 @@ module.exports = function (opts) {
     ssb.add(
       prev = create(keys, null, {type: 'init', public: keys.public}),
       function (err) {
-        if(err) throw err
+        if(err) throw explain(err, 'init failed')
 
         ssb.add(
           prev = create(keys, 'msg', 'hello', prev),
           function (err) {
-            if(err) throw err
+            if(err) throw explain(err, 'hello failed')
 
             ssb.add(
               prev = create(keys, 'msg', 'hello2', prev),
               function (err) {
-                if(err) throw err
+                if(err) throw explain(err, 'hello2 failed')
                 pull(
-                  db.createFeedStream(id),
+                  db.createFeedStream(),
                   pull.collect(function (err, ary) {
-                    if(err) throw err
+                    if(err) throw explain(err, 'createFeedStream failed')
                     t.deepEqual(ary.pop(), prev)
                     t.end()
                   })

From 349c5eedd40af6a500558e0add97468f205e51fa Mon Sep 17 00:00:00 2001
From: Dominic Tarr <dominic.tarr@gmail.com>
Date: Tue, 18 Nov 2014 13:50:04 +0100
Subject: [PATCH 15/16] support string hashes in test/defaults

---
 test/defaults.js | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/test/defaults.js b/test/defaults.js
index 5aecdd34..cd820a15 100644
--- a/test/defaults.js
+++ b/test/defaults.js
@@ -7,11 +7,31 @@ function copy (buf) {
   return _buf
 }
 
+function isString(s) {
+  return 'string' === typeof s
+}
+
+function isHash (data) {
+  return isString(data) && /^[A-Za-z0-9\/+]{43}=\.blake2s$/.test(data)
+  //return Buffer.isBuffer(data) && data.length == 32
+}
+
 function randint (n) {
   return ~~(Math.random()*n)
 }
 
+function decodeHash(hash) {
+  if(!isHash(hash)) throw new Error('sign expects a hash')
+  return new Buffer(hash.substring(0, 44), 'base64')
+}
+
+function encodeHash (buf) {
+  return buf.toString('base64') +'.blake2s'
+}
+
 function flipRandomBit(buf) {
+    if(isHash(buf))
+      return encodeHash(flipRandomBit(decodeHash(buf)))
     buf = copy(buf)
     var r = randint(buf.length)
     //change one bit
@@ -90,6 +110,7 @@ module.exports = function (opts) {
 
     //should this throw?
     t.ok(validation.validateSync(msg, null, keys), 'initial message')
+    t.equal(validation.validateSync.reason, '')
     t.end()
   })
 

From c38d600cdbba7f355a50273c32e265ee56dc99ea Mon Sep 17 00:00:00 2001
From: Dominic Tarr <dominic.tarr@gmail.com>
Date: Tue, 18 Nov 2014 13:50:15 +0100
Subject: [PATCH 16/16] use explain-error

---
 package.json | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/package.json b/package.json
index 5c1eeb2a..20134f7f 100644
--- a/package.json
+++ b/package.json
@@ -59,7 +59,8 @@
     "rimraf": "~2.2.8",
     "multicb": "~0.0.2",
     "hexpp": "~1.1.3",
-    "pull-randomly-split": "~1.0.4"
+    "pull-randomly-split": "~1.0.4",
+    "explain-error": "~1.0.0"
   },
   "scripts": {
     "test": "set -e; for t in test/*.js; do node $t; done"
@@ -81,4 +82,4 @@
       "iphone/6.0..latest"
     ]
   }
-}
+}
\ No newline at end of file