fix: remove node buffer (#43)

Swaps node Buffer for Uint8Array in keys and values.

BREAKING CHANGES:

- node Buffers have been replaced with Uint8Arrays
- `key.toBuffer` has been replaced with `key.uint8Array()`

Author: achingbrain

.npmignore

@@ -1,7 +1,17 @@
+yarn.lock
+package-lock.json
+
+**/node_modules/
+**/*.log
+test/repo-tests*
+
 # Logs
 logs
 *.log
 
+coverage
+.nyc_output
+
 # Runtime data
 pids
 *.pid
@@ -12,19 +22,20 @@ lib-cov
 
 # Coverage directory used by tools like istanbul
 coverage
-.nyc_output
 
 # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
 .grunt
 
 # node-waf configuration
 .lock-wscript
 
-# Compiled binary addons (http://nodejs.org/api/addons.html)
-build/Release
+build
 
 # Dependency directory
 # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git
 node_modules
 
-test
+.travis.yml
+.github
+docs
+test

README.md

@@ -33,13 +33,13 @@
   - [`put(key, value, [options])` -> `Promise`](#putkey-value-options---promise)
     - [Arguments](#arguments-1)
     - [Example](#example-1)
-  - [`putMany(source, [options])` -> `AsyncIterator<{ key: Key, value: Buffer }>`](#putmanysource-options---asynciterator-key-key-value-buffer-)
+  - [`putMany(source, [options])` -> `AsyncIterator<{ key: Key, value: Uint8Array }>`](#putmanysource-options---asynciterator-key-key-value-uint8array-)
     - [Arguments](#arguments-2)
     - [Example](#example-2)
-  - [`get(key, [options])` -> `Promise<Buffer>`](#getkey-options---promisebuffer)
+  - [`get(key, [options])` -> `Promise<Uint8Array>`](#getkey-options---promiseuint8array)
     - [Arguments](#arguments-3)
     - [Example](#example-3)
-  - [`getMany(source, [options])` -> `AsyncIterator<Buffer>`](#getmanysource-options---asynciteratorbuffer)
+  - [`getMany(source, [options])` -> `AsyncIterator<Uint8Array>`](#getmanysource-options---asynciteratoruint8array)
     - [Arguments](#arguments-4)
     - [Example](#example-4)
   - [`delete(key, [options])` -> `Promise`](#deletekey-options---promise)
@@ -48,7 +48,7 @@
   - [`deleteMany(source, [options])` -> `AsyncIterator<Key>`](#deletemanysource-options---asynciteratorkey)
     - [Arguments](#arguments-6)
     - [Example](#example-6)
-  - [`query(query, [options])` -> `AsyncIterable<Buffer>`](#queryquery-options---asynciterablebuffer)
+  - [`query(query, [options])` -> `AsyncIterable<Uint8Array>`](#queryquery-options---asynciterableuint8array)
     - [Arguments](#arguments-7)
     - [Example](#example-7)
   - [`batch()`](#batch)
@@ -179,11 +179,11 @@ for await (const { key, data } of batch(store.putMany(source), 10)) {
 
 ### Keys
 
-To allow a better abstraction on how to address values, there is a `Key` class which is used as identifier. It's easy to create a key from a `Buffer` or a `string`.
+To allow a better abstraction on how to address values, there is a `Key` class which is used as identifier. It's easy to create a key from a `Uint8Array` or a `string`.
 
 ```js
 const a = new Key('a')
-const b = new Key(Buffer.from('hello'))
+const b = new Key(new Uint8Array([0, 1, 2, 3]))
 ```
 
 The key scheme is inspired by file systems and Google App Engine key model. Keys are meant to be unique across a system. They are typically hierarchical, incorporating more and more specific namespaces. Thus keys can be deemed 'children' or 'ancestors' of other keys:
@@ -234,41 +234,41 @@ Store a value with the given key.
 | Name | Type | Description |
 | ---- | ---- | ----------- |
 | key | [Key][] | The key to store the value under |
-| value | [Buffer][] | Value to store |
+| value | [Uint8Array][] | Value to store |
 | options | [Object][] | An options object, all properties are optional |
 | options.signal | [AbortSignal][] | A way to signal that the caller is no longer interested in the outcome of this operation |
 
 #### Example
 
 ```js
-await store.put([{ key: new Key('awesome'), value: Buffer.from('datastores') }])
+await store.put([{ key: new Key('awesome'), value: new Uint8Array([0, 1, 2, 3]) }])
 console.log('put content')
 ```
 
-### `putMany(source, [options])` -> `AsyncIterator<{ key: Key, value: Buffer }>`
+### `putMany(source, [options])` -> `AsyncIterator<{ key: Key, value: Uint8Array }>`
 
 Store many key-value pairs.
 
 #### Arguments
 
 | Name | Type | Description |
 | ---- | ---- | ----------- |
-| source | [AsyncIterator][]<{ key: [Key][], value: [Buffer][] }> | The key to store the value under |
-| value | [Buffer][] | Value to store |
+| source | [AsyncIterator][]<{ key: [Key][], value: [Uint8Array][] }> | The key to store the value under |
+| value | [Uint8Array][] | Value to store |
 | options | [Object][] | An options object, all properties are optional |
 | options.signal | [AbortSignal][] | A way to signal that the caller is no longer interested in the outcome of this operation |
 
 #### Example
 
 ```js
-const source = [{ key: new Key('awesome'), value: Buffer.from('datastores') }]
+const source = [{ key: new Key('awesome'), value: new Uint8Array([0, 1, 2, 3]) }]
 
 for await (const { key, value } of store.putMany(source)) {
   console.info(`put content for key ${key}`)
 }
 ```
 
-### `get(key, [options])` -> `Promise<Buffer>`
+### `get(key, [options])` -> `Promise<Uint8Array>`
 
 #### Arguments
 
@@ -288,7 +288,7 @@ console.log('got content: %s', value.toString('utf8'))
 // => got content: datastore
 ```
 
-### `getMany(source, [options])` -> `AsyncIterator<Buffer>`
+### `getMany(source, [options])` -> `AsyncIterator<Uint8Array>`
 
 #### Arguments
 
@@ -304,7 +304,7 @@ Retrieve a stream of values stored under the given keys.
 
 ```js
 for await (const value of store.getMany([new Key('awesome')])) {
-  console.log('got content: %s', value.toString('utf8'))
+  console.log('got content:', new TextDecoder('utf8').decode(value))
   // => got content: datastore
 }
 ```
@@ -350,18 +350,18 @@ for await (const key of store.deleteMany(source)) {
 }
 ```
 
-### `query(query, [options])` -> `AsyncIterable<Buffer>`
+### `query(query, [options])` -> `AsyncIterable<Uint8Array>`
 
-Search the store for some values. Returns an [AsyncIterable][] with each item being a [Buffer][].
+Search the store for some values. Returns an [AsyncIterable][] with each item being a [Uint8Array][].
 
 #### Arguments
 
 | Name | Type | Description |
 | ---- | ---- | ----------- |
 | query | [Object][] | A query object, all properties are optional |
 | query.prefix | [String][] | Only return values where the key starts with this prefix |
-| query.filters | [Array][]<[Function][]([Buffer][]) -> [Boolean][]> | Filter the results according to the these functions |
-| query.orders | [Array][]<[Function][]([Array][]<[Buffer][]>) -> [Array][]<[Buffer][]>> | Order the results according to these functions |
+| query.filters | [Array][]<[Function][]([Uint8Array][]) -> [Boolean][]> | Filter the results according to the these functions |
+| query.orders | [Array][]<[Function][]([Array][]<[Uint8Array][]>) -> [Array][]<[Uint8Array][]>> | Order the results according to these functions |
 | query.limit | [Number][] | Only return this many records |
 | query.offset | [Number][] | Skip this many records at the beginning |
 | options | [Object][] | An options object, all properties are optional |
@@ -388,7 +388,7 @@ This will return an object with which you can chain multiple operations together
 const b = store.batch()
 
 for (let i = 0; i < 100; i++) {
-  b.put(new Key(`hello${i}`), Buffer.from(`hello world ${i}`))
+  b.put(new Key(`hello${i}`), new TextEncoder('utf8').encode(`hello world ${i}`))
 }
 
 await b.commit()
@@ -402,7 +402,7 @@ Queue a put operation to the store.
 | Name | Type | Description |
 | ---- | ---- | ----------- |
 | key | [Key][] | The key to store the value under |
-| value | [Buffer][] | Value to store |
+| value | [Uint8Array][] | Value to store |
 
 #### `delete(key)`
 
@@ -428,7 +428,7 @@ Write all queued operations to the underyling store. The batch object should not
 ```js
 const batch = store.batch()
 
-batch.put(new Key('to-put'), Buffer.from('hello world'))
+batch.put(new Key('to-put'), new TextEncoder('utf8').encode('hello world'))
 batch.del(new Key('to-remove'))
 
 await batch.commit()
@@ -455,7 +455,7 @@ MIT 2017 © IPFS
 
 [Key]: #Keys
 [Object]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object
-[Buffer]: https://nodejs.org/api/buffer.html
+[Uint8Array]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Uint8Array
 [AbortSignal]: https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal
 [AsyncIterator]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Symbol/asyncIterator
 [AsyncIterable]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Iteration_protocols

package.json

@@ -33,15 +33,14 @@
   },
   "homepage": "https://github.com/ipfs/interface-datastore#readme",
   "devDependencies": {
-    "aegir": "^22.0.0",
+    "aegir": "^25.0.0",
     "chai": "^4.1.2",
     "dirty-chai": "^2.0.1"
   },
   "dependencies": {
-    "buffer": "^5.5.0",
     "class-is": "^1.1.0",
     "err-code": "^2.0.1",
-    "ipfs-utils": "^2.2.2",
+    "ipfs-utils": "^2.3.1",
     "iso-random-stream": "^1.1.1",
     "it-all": "^1.0.2",
     "it-drain": "^1.0.1",

src/adapter.js

@@ -16,7 +16,7 @@ class InterfaceDatastoreAdapter {
    * Store the passed value under the passed key
    *
    * @param {Key} key
-   * @param {Buffer} val
+   * @param {Uint8Array} val
    * @param {Object} options
    * @returns {Promise<void>}
    */
@@ -27,9 +27,9 @@ class InterfaceDatastoreAdapter {
   /**
    * Store the given key/value pairs
    *
-   * @param {AsyncIterator<{ key: Key, value: Buffer }>} source
+   * @param {AsyncIterator<{ key: Key, value: Uint8Array }>} source
    * @param {Object} options
-   * @returns {AsyncIterator<{ key: Key, value: Buffer }>}
+   * @returns {AsyncIterator<{ key: Key, value: Uint8Array }>}
    */
   async * putMany (source, options = {}) {
     for await (const { key, value } of source) {
@@ -43,7 +43,7 @@ class InterfaceDatastoreAdapter {
    *
    * @param {Key} key
    * @param {Object} options
-   * @returns {Promise<Buffer>}
+   * @returns {Promise<Uint8Array>}
    */
   async get (key, options = {}) { // eslint-disable-line require-await
 
@@ -54,7 +54,7 @@ class InterfaceDatastoreAdapter {
    *
    * @param {AsyncIterator<Key>} source
    * @param {Object} options
-   * @returns {AsyncIterator<Buffer>}
+   * @returns {AsyncIterator<Uint8Array>}
    */
   async * getMany (source, options = {}) {
     for await (const key of source) {
@@ -127,7 +127,7 @@ class InterfaceDatastoreAdapter {
    *
    * @param {Object} q
    * @param {Object} options
-   * @returns {AsyncIterable<{ key: Key, value: Buffer }>}
+   * @returns {AsyncIterable<{ key: Key, value: Uint8Array }>}
    */
   async * _all (q, options) { // eslint-disable-line require-await
 
@@ -138,7 +138,7 @@ class InterfaceDatastoreAdapter {
    *
    * @param {Object} q
    * @param {Object} options
-   * @returns {AsyncIterable<Buffer>}
+   * @returns {AsyncIterable<Uint8Array>}
    */
   async * query (q, options) { // eslint-disable-line require-await
     let it = this._all(q, options)

src/key.js

@@ -1,11 +1,12 @@
 'use strict'
 
-const { Buffer } = require('buffer')
 const { nanoid } = require('nanoid')
 const withIs = require('class-is')
+const { utf8Encoder, utf8Decoder } = require('./utils')
+const TextDecoder = require('ipfs-utils/src/text-decoder')
 
 const pathSepS = '/'
-const pathSepB = Buffer.from(pathSepS)
+const pathSepB = utf8Encoder.encode(pathSepS)
 const pathSep = pathSepB[0]
 
 /**
@@ -27,9 +28,11 @@ const pathSep = pathSepB[0]
 class Key {
   constructor (s, clean) {
     if (typeof s === 'string') {
-      this._buf = Buffer.from(s)
-    } else if (Buffer.isBuffer(s)) {
+      this._buf = utf8Encoder.encode(s)
+    } else if (s instanceof Uint8Array) {
       this._buf = s
+    } else {
+      throw new Error('Invalid key, should be String of Uint8Array')
     }
 
     if (clean == null) {
@@ -40,7 +43,7 @@ class Key {
       this.clean()
     }
 
-    if (this._buf.length === 0 || this._buf[0] !== pathSep) {
+    if (this._buf.byteLength === 0 || this._buf[0] !== pathSep) {
       throw new Error('Invalid key')
     }
   }
@@ -51,16 +54,20 @@ class Key {
    * @param {string} [encoding='utf8']
    * @returns {string}
    */
-  toString (encoding) {
-    return this._buf.toString(encoding || 'utf8')
+  toString (encoding = 'utf8') {
+    if (encoding === 'utf8' || encoding === 'utf-8') {
+      return utf8Decoder.decode(this._buf)
+    }
+
+    return new TextDecoder(encoding).decode(this._buf)
   }
 
   /**
-   * Return the buffer representation of the key
+   * Return the Uint8Array representation of the key
    *
-   * @returns {Buffer}
+   * @returns {Uint8Array}
    */
-  toBuffer () {
+  uint8Array () {
     return this._buf
   }
 
@@ -106,17 +113,20 @@ class Key {
    * @returns {void}
    */
   clean () {
-    if (!this._buf || this._buf.length === 0) {
-      this._buf = Buffer.from(pathSepS)
+    if (!this._buf || this._buf.byteLength === 0) {
+      this._buf = pathSepB
     }
 
     if (this._buf[0] !== pathSep) {
-      this._buf = Buffer.concat([pathSepB, this._buf])
+      const bytes = new Uint8Array(this._buf.byteLength + 1)
+      bytes.fill(pathSep, 0, 1)
+      bytes.set(this._buf, 1)
+      this._buf = bytes
     }
 
     // normalize does not remove trailing slashes
-    while (this._buf.length > 1 && this._buf[this._buf.length - 1] === pathSep) {
-      this._buf = this._buf.slice(0, -1)
+    while (this._buf.byteLength > 1 && this._buf[this._buf.byteLength - 1] === pathSep) {
+      this._buf = this._buf.subarray(0, -1)
     }
   }