From 9cfbc120dda78b6a14a3e9a303ab562f22405edb Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 3 Aug 2020 18:03:11 +0100 Subject: [PATCH 1/3] fix: replace node buffers with uint8arrays BREAKING CHANGE: - `util.serialize` now returns a Uint8Array --- README.md | 31 +++++++++++++++++++------------ package.json | 6 +++--- src/resolver.js | 7 +++---- src/util.js | 17 +++++++++-------- test/util.spec.js | 13 +++++++------ 5 files changed, 41 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 9fafb05..a0f8568 100644 --- a/README.md +++ b/README.md @@ -20,15 +20,22 @@ ## Table of Contents -- [Install](#install) - - [npm](#npm) - - [Use in Node.js](#use-in-nodejs) - - [Use in a browser with browserify, webpack or any other bundler](#use-in-a-browser-with-browserify-webpack-or-any-other-bundler) - - [Use in a browser Using a script tag](#use-in-a-browser-using-a-script-tag) -- [Usage](#usage) -- [API](#api) -- [Contribute](#contribute) -- [License](#license) +- [js-ipld-dag-cbor](#js-ipld-dag-cbor) + - [Lead Maintainer](#lead-maintainer) + - [Table of Contents](#table-of-contents) + - [Install](#install) + - [npm](#npm) + - [Use in Node.js](#use-in-nodejs) + - [Use in a browser with browserify, webpack or any other bundler](#use-in-a-browser-with-browserify-webpack-or-any-other-bundler) + - [Use in a browser Using a script tag](#use-in-a-browser-using-a-script-tag) + - [Usage](#usage) + - [API](#api) + - [`dagCBOR.util.serialize(obj)`](#dagcborutilserializeobj) + - [`dagCBOR.util.deserialize(serialized)`](#dagcborutildeserializeserialized) + - [`dagCBOR.util.configureDecoder([options])`](#dagcborutilconfiguredecoderoptions) + - [`dagCBOR.util.cid(obj[, options,])`](#dagcborutilcidobj-options) + - [Contribute](#contribute) + - [License](#license) ## Install @@ -73,13 +80,13 @@ const file = { } const serialized = dagCBOR.util.serialize(file) -console.log(`Encoded as a ${serialized.length} byte Buffer`) +console.log(`Encoded as a ${serialized.length} byte Uint8Array`) const node = dagCBOR.util.deserialize(serialized) console.log('Decoded as:', node) require('assert').deepEqual(node, file) // should match -// → Encoded as a 22 byte Buffer +// → Encoded as a 22 byte Uint8Array // → Decoded as: { name: 'hello.txt', size: 11 } ``` @@ -97,7 +104,7 @@ Returns the serialized node. Decodes an IPLD CBOR encoded representation, restoring any CBOR tags (id `42`) to CIDs. - - `serialized` (`Buffer` or `String`): a binary blob representing an IPLD CBOR encoded object. + - `serialized` (`Uint8Array` or `String`): a binary blob representing an IPLD CBOR encoded object. Returns the deserialized object. diff --git a/package.json b/package.json index d99f6a2..a77852f 100644 --- a/package.json +++ b/package.json @@ -38,11 +38,11 @@ "homepage": "https://github.com/ipld/js-ipld-dag-cbor", "dependencies": { "borc": "^2.1.2", - "buffer": "^5.6.0", - "cids": "~0.8.3", + "cids": "ipld/js-cid#fix/support-uint8arrays", "is-circular": "^1.0.2", "multicodec": "^1.0.3", - "multihashing-async": "^1.0.0" + "multihashing-async": "^1.0.0", + "uint8arrays": "^1.0.0" }, "devDependencies": { "aegir": "^25.0.0", diff --git a/src/resolver.js b/src/resolver.js index ab5e458..3735648 100644 --- a/src/resolver.js +++ b/src/resolver.js @@ -1,7 +1,6 @@ 'use strict' const CID = require('cids') -const { Buffer } = require('buffer') const util = require('./util') /** @@ -10,7 +9,7 @@ const util = require('./util') * Returns the value or a link and the partial mising path. This way the * IPLD Resolver can fetch the link and continue to resolve. * - * @param {Buffer} binaryBlob - Binary representation of a CBOR block + * @param {Uint8Array} binaryBlob - Binary representation of a CBOR block * @param {string} [path='/'] - Path that should be resolved * @returns {Object} result - Result of the path it it was resolved successfully * @returns {*} result.value - Value the path resolves to @@ -45,7 +44,7 @@ exports.resolve = (binaryBlob, path) => { const traverse = function * (node, path) { // Traverse only objects and arrays - if (Buffer.isBuffer(node) || CID.isCID(node) || typeof node === 'string' || + if (node instanceof Uint8Array || CID.isCID(node) || typeof node === 'string' || node === null) { return } @@ -60,7 +59,7 @@ const traverse = function * (node, path) { * Return all available paths of a block. * * @generator - * @param {Buffer} binaryBlob - Binary representation of a CBOR block + * @param {Uint8Array} binaryBlob - Binary representation of a CBOR block * @yields {string} - A single path */ exports.tree = function * (binaryBlob) { diff --git a/src/util.js b/src/util.js index 8d3125b..89b862c 100644 --- a/src/util.js +++ b/src/util.js @@ -1,26 +1,27 @@ 'use strict' const cbor = require('borc') -const { Buffer } = require('buffer') const multicodec = require('multicodec') const multihashing = require('multihashing-async') const CID = require('cids') const isCircular = require('is-circular') +const uint8ArrayConcat = require('uint8arrays/concat') +const uint8ArrayFromString = require('uint8arrays/from-string') // https://github.com/ipfs/go-ipfs/issues/3570#issuecomment-273931692 const CID_CBOR_TAG = 42 function tagCID (cid) { if (typeof cid === 'string') { - cid = new CID(cid).buffer + cid = new CID(cid).bytes } else if (CID.isCID(cid)) { - cid = cid.buffer + cid = cid.bytes } - return new cbor.Tagged(CID_CBOR_TAG, Buffer.concat([ - Buffer.from('00', 'hex'), // thanks jdag + return new cbor.Tagged(CID_CBOR_TAG, uint8ArrayConcat([ + uint8ArrayFromString('00', 'base16'), // thanks jdag cid - ])) + ], 1 + cid.length)) } function replaceCIDbyTAG (dagNode) { @@ -129,7 +130,7 @@ exports.configureDecoder() // Setup default cbor.Decoder * Serialize internal representation into a binary CBOR block. * * @param {Object} node - Internal representation of a CBOR block - * @returns {Buffer} - The encoded binary representation + * @returns {Uint8Array} - The encoded binary representation */ exports.serialize = (node) => { const nodeTagged = replaceCIDbyTAG(node) @@ -141,7 +142,7 @@ exports.serialize = (node) => { /** * Deserialize CBOR block into the internal representation. * - * @param {Buffer} data - Binary representation of a CBOR block + * @param {Uint8Array} data - Binary representation of a CBOR block * @returns {Object} - An object that conforms to the IPLD Data Model */ exports.deserialize = (data) => { diff --git a/test/util.spec.js b/test/util.spec.js index 37c3ea0..6e898d8 100644 --- a/test/util.spec.js +++ b/test/util.spec.js @@ -1,7 +1,6 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') const chai = require('chai') const dirtyChai = require('dirty-chai') const expect = chai.expect @@ -10,6 +9,8 @@ const garbage = require('garbage') const dagCBOR = require('../src') const multihash = require('multihashes') const CID = require('cids') +const uint8ArrayFromString = require('uint8arrays/from-string') +const uint8ArrayConcat = require('uint8arrays/concat') describe('util', () => { const obj = { @@ -27,7 +28,7 @@ describe('util', () => { const serializedObj = dagCBOR.util.serialize(obj) it('.serialize and .deserialize', () => { - expect(Buffer.isBuffer(serializedObj)).to.equal(true) + expect(serializedObj).to.be.a('Uint8Array') // Check for the tag 42 // d8 = tag, 2a = 42 @@ -45,7 +46,7 @@ describe('util', () => { const largeObj = { someKey: [].slice.call(new Uint8Array(dataSize)) } const serialized = dagCBOR.util.serialize(largeObj) - expect(Buffer.isBuffer(serialized)).to.be.true() + expect(serialized).to.be.a('Uint8Array') const deserialized = dagCBOR.util.deserialize(serialized) expect(largeObj).to.eql(deserialized) @@ -60,7 +61,7 @@ describe('util', () => { dagCBOR.util.configureDecoder({ size: 64 * 1024, maxSize: 128 * 1024 }) // 64 Kb start, 128 Kb max const serialized = dagCBOR.util.serialize(largeObj) - expect(Buffer.isBuffer(serialized)).to.be.true() + expect(serialized).to.be.a('Uint8Array') expect(() => dagCBOR.util.deserialize(serialized)).to.throw( 'Data is too large to deserialize with current decoder') @@ -111,7 +112,7 @@ describe('util', () => { }) it('.serialize and .deserialize object with Uint8Array field', () => { - const buffer = Buffer.from('some data') + const buffer = uint8ArrayFromString('some data') const bytes = Uint8Array.from(buffer) const s1 = dagCBOR.util.serialize({ data: buffer }) @@ -127,7 +128,7 @@ describe('util', () => { expect(() => // two top-level CBOR objects, the original and a single uint=0, valid if using // CBOR in streaming mode, not valid here - dagCBOR.util.deserialize(Buffer.concat([serializedObj, Buffer.alloc(1)])) + dagCBOR.util.deserialize(uint8ArrayConcat([serializedObj, new Uint8Array(1)])) ).to.throw(Error, 'Extraneous CBOR data found beyond initial top-level object') }) }) From ed09b6ef36fff14eb4f3b59e05c22d46185d4a6a Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 4 Aug 2020 13:46:58 +0100 Subject: [PATCH 2/3] chore: update deps --- package.json | 10 ++++------ test/interop.spec.js | 5 +---- test/mod.spec.js | 6 +----- test/resolver.spec.js | 7 +------ test/util.spec.js | 5 +---- 5 files changed, 8 insertions(+), 25 deletions(-) diff --git a/package.json b/package.json index a77852f..aabe52b 100644 --- a/package.json +++ b/package.json @@ -38,19 +38,17 @@ "homepage": "https://github.com/ipld/js-ipld-dag-cbor", "dependencies": { "borc": "^2.1.2", - "cids": "ipld/js-cid#fix/support-uint8arrays", + "cids": "^1.0.0", "is-circular": "^1.0.2", - "multicodec": "^1.0.3", - "multihashing-async": "^1.0.0", + "multicodec": "^2.0.0", + "multihashing-async": "^2.0.0", "uint8arrays": "^1.0.0" }, "devDependencies": { "aegir": "^25.0.0", - "chai": "^4.2.0", "detect-node": "^2.0.4", - "dirty-chai": "^2.0.1", "garbage": "0.0.0", - "multihashes": "^1.0.1" + "multihashes": "^3.0.1" }, "contributors": [ "David Dias ", diff --git a/test/interop.spec.js b/test/interop.spec.js index 7ac4ccd..b2d3f33 100644 --- a/test/interop.spec.js +++ b/test/interop.spec.js @@ -1,10 +1,7 @@ /* eslint-env mocha */ 'use strict' -const chai = require('chai') -const dirtyChai = require('dirty-chai') -const expect = chai.expect -chai.use(dirtyChai) +const { expect } = require('aegir/utils/chai') const dagCBOR = require('../src') const loadFixture = require('aegir/fixtures') const isNode = require('detect-node') diff --git a/test/mod.spec.js b/test/mod.spec.js index e13f912..00cbae9 100644 --- a/test/mod.spec.js +++ b/test/mod.spec.js @@ -1,12 +1,8 @@ /* eslint-env mocha */ 'use strict' -const chai = require('chai') -const dirtyChai = require('dirty-chai') -const expect = chai.expect -chai.use(dirtyChai) +const { expect } = require('aegir/utils/chai') const multicodec = require('multicodec') - const mod = require('../src') describe('IPLD Format', () => { diff --git a/test/resolver.spec.js b/test/resolver.spec.js index 5fbdeec..f02dc39 100644 --- a/test/resolver.spec.js +++ b/test/resolver.spec.js @@ -2,13 +2,8 @@ /* eslint max-nested-callbacks: ["error", 5] */ 'use strict' -const chai = require('chai') -const dirtyChai = require('dirty-chai') -const expect = chai.expect -chai.use(dirtyChai) - +const { expect } = require('aegir/utils/chai') const CID = require('cids') - const dagCBOR = require('../src') const resolver = dagCBOR.resolver diff --git a/test/util.spec.js b/test/util.spec.js index 6e898d8..1b67d34 100644 --- a/test/util.spec.js +++ b/test/util.spec.js @@ -1,10 +1,7 @@ /* eslint-env mocha */ 'use strict' -const chai = require('chai') -const dirtyChai = require('dirty-chai') -const expect = chai.expect -chai.use(dirtyChai) +const { expect } = require('aegir/utils/chai') const garbage = require('garbage') const dagCBOR = require('../src') const multihash = require('multihashes') From 41cb24e0e9c321f69696182dd0895e005a6a22d4 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 4 Aug 2020 14:04:15 +0100 Subject: [PATCH 3/3] chore: update readme --- README.md | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index a0f8568..a446c3c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# js-ipld-dag-cbor +# js-ipld-dag-cbor [![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) [![](https://img.shields.io/badge/project-IPLD-blue.svg?style=flat-square)](http://github.com/ipld/ipld) @@ -14,28 +14,25 @@ > JavaScript implementation of the [IPLD spec](https://github.com/ipfs/specs/tree/master/ipld). -## Lead Maintainer +## Lead Maintainer [Volker Mische](https://github.com/vmx) -## Table of Contents - -- [js-ipld-dag-cbor](#js-ipld-dag-cbor) - - [Lead Maintainer](#lead-maintainer) - - [Table of Contents](#table-of-contents) - - [Install](#install) - - [npm](#npm) - - [Use in Node.js](#use-in-nodejs) - - [Use in a browser with browserify, webpack or any other bundler](#use-in-a-browser-with-browserify-webpack-or-any-other-bundler) - - [Use in a browser Using a script tag](#use-in-a-browser-using-a-script-tag) - - [Usage](#usage) - - [API](#api) - - [`dagCBOR.util.serialize(obj)`](#dagcborutilserializeobj) - - [`dagCBOR.util.deserialize(serialized)`](#dagcborutildeserializeserialized) - - [`dagCBOR.util.configureDecoder([options])`](#dagcborutilconfiguredecoderoptions) - - [`dagCBOR.util.cid(obj[, options,])`](#dagcborutilcidobj-options) - - [Contribute](#contribute) - - [License](#license) +## Table of Contents + +- [Install](#install) + - [npm](#npm) + - [Use in Node.js](#use-in-nodejs) + - [Use in a browser with browserify, webpack or any other bundler](#use-in-a-browser-with-browserify-webpack-or-any-other-bundler) + - [Use in a browser Using a script tag](#use-in-a-browser-using-a-script-tag) +- [Usage](#usage) +- [API](#api) + - [`dagCBOR.util.serialize(obj)`](#dagcborutilserializeobj) + - [`dagCBOR.util.deserialize(serialized)`](#dagcborutildeserializeserialized) + - [`dagCBOR.util.configureDecoder([options])`](#dagcborutilconfiguredecoderoptions) + - [`dagCBOR.util.cid(obj[, options,])`](#dagcborutilcidobj-options) +- [Contribute](#contribute) +- [License](#license) ## Install