From 4705cb0a9de5295ee71aa419ba300f15de85ed0a Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Wed, 7 Apr 2021 20:34:36 -0700 Subject: [PATCH 1/6] test(coreapi): block.get + dag.get for identity hash Add core interface tests for block.get and dag.get when fetching identity hashes --- packages/interface-ipfs-core/src/block/get.js | 12 ++++++++++-- packages/interface-ipfs-core/src/dag/get.js | 9 +++++++++ packages/ipfs-cli/package.json | 2 +- packages/ipfs-core/package.json | 2 +- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/packages/interface-ipfs-core/src/block/get.js b/packages/interface-ipfs-core/src/block/get.js index 7ab52f83ff..c9790daa1f 100644 --- a/packages/interface-ipfs-core/src/block/get.js +++ b/packages/interface-ipfs-core/src/block/get.js @@ -2,7 +2,7 @@ 'use strict' const uint8ArrayFromString = require('uint8arrays/from-string') -const multihash = require('multihashing-async').multihash +const multihashing = require('multihashing-async') const CID = require('cids') const { getDescribe, getIt, expect } = require('../utils/mocha') const testTimeout = require('../utils/test-timeout') @@ -43,7 +43,7 @@ module.exports = (common, options) => { }) it('should get by CID in string', async () => { - const block = await ipfs.block.get(multihash.toB58String(hash)) + const block = await ipfs.block.get(multihashing.multihash.toB58String(hash)) expect(block.data).to.eql(uint8ArrayFromString('blorb')) expect(block.cid.multihash).to.eql(hash) @@ -89,6 +89,14 @@ module.exports = (common, options) => { expect(block.data).to.eql(input) }) + it('should get a block with an identity CID, without putting first', async () => { + const identityData = uint8ArrayFromString('A16461736466190144', 'base16upper') + const identityHash = await multihashing(identityData, 'identity') + const identityCID = new CID(1, 'dag-cbor', identityHash) + const block = await ipfs.block.get(identityCID) + expect(block.data).to.eql(identityData) + }) + it('should return an error for an invalid CID', () => { return expect(ipfs.block.get('Non-base58 character')).to.eventually.be.rejected .and.be.an.instanceOf(Error) diff --git a/packages/interface-ipfs-core/src/dag/get.js b/packages/interface-ipfs-core/src/dag/get.js index 4bff6a9528..b0a4d521a8 100644 --- a/packages/interface-ipfs-core/src/dag/get.js +++ b/packages/interface-ipfs-core/src/dag/get.js @@ -11,6 +11,7 @@ const all = require('it-all') const CID = require('cids') const { getDescribe, getIt, expect } = require('../utils/mocha') const testTimeout = require('../utils/test-timeout') +const multihashing = require('multihashing-async') /** @typedef { import("ipfsd-ctl/src/factory") } Factory */ /** @@ -240,6 +241,14 @@ module.exports = (common, options) => { expect(result.value).to.deep.equal(buf) }) + it('should be able to get a dag-cbor node with the identity hash', async() => { + const identityData = uint8ArrayFromString('A16461736466190144', 'base16upper') + const identityHash = await multihashing(identityData, 'identity') + const identityCID = new CID(1, 'dag-cbor', identityHash) + const result = await ipfs.dag.get(identityCID) + expect(result.value).to.deep.equal({"asdf":324}) + }) + it('should throw error for invalid string CID input', () => { return expect(ipfs.dag.get('INVALID CID')) .to.eventually.be.rejected() diff --git a/packages/ipfs-cli/package.json b/packages/ipfs-cli/package.json index 4359a3ecf0..eefcf8d53f 100644 --- a/packages/ipfs-cli/package.json +++ b/packages/ipfs-cli/package.json @@ -42,7 +42,7 @@ "ipfs-core-utils": "^0.7.2", "ipfs-daemon": "^0.5.4", "ipfs-http-client": "^49.0.4", - "ipfs-repo": "^9.0.0", + "ipfs-repo": "github:hannahhoward/js-ipfs-repo#d0ad9a4", "ipfs-utils": "^6.0.4", "ipld-dag-cbor": "^0.18.0", "ipld-dag-pb": "^0.22.1", diff --git a/packages/ipfs-core/package.json b/packages/ipfs-core/package.json index 8f26e64c8e..e0d87c166e 100644 --- a/packages/ipfs-core/package.json +++ b/packages/ipfs-core/package.json @@ -74,7 +74,7 @@ "ipfs-block-service": "^0.19.0", "ipfs-core-types": "^0.3.1", "ipfs-core-utils": "^0.7.2", - "ipfs-repo": "^9.0.0", + "ipfs-repo": "github:hannahhoward/js-ipfs-repo#d0ad9a4", "ipfs-unixfs": "^4.0.1", "ipfs-unixfs-exporter": "^5.0.1", "ipfs-unixfs-importer": "^7.0.1", From e2da92aed15ed797b35a7e75800c2ddaeb56c70e Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Thu, 8 Apr 2021 05:35:00 -0700 Subject: [PATCH 2/6] fix(http-server): properly stream UInt8Array make sure that UInt8Arrays are converted to Buffer so Hapi properly sends them as binary --- packages/interface-ipfs-core/src/dag/get.js | 4 ++-- packages/ipfs-http-server/src/api/resources/block.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/interface-ipfs-core/src/dag/get.js b/packages/interface-ipfs-core/src/dag/get.js index b0a4d521a8..cc7f7803e6 100644 --- a/packages/interface-ipfs-core/src/dag/get.js +++ b/packages/interface-ipfs-core/src/dag/get.js @@ -241,12 +241,12 @@ module.exports = (common, options) => { expect(result.value).to.deep.equal(buf) }) - it('should be able to get a dag-cbor node with the identity hash', async() => { + it('should be able to get a dag-cbor node with the identity hash', async () => { const identityData = uint8ArrayFromString('A16461736466190144', 'base16upper') const identityHash = await multihashing(identityData, 'identity') const identityCID = new CID(1, 'dag-cbor', identityHash) const result = await ipfs.dag.get(identityCID) - expect(result.value).to.deep.equal({"asdf":324}) + expect(result.value).to.deep.equal({ asdf: 324 }) }) it('should throw error for invalid string CID input', () => { diff --git a/packages/ipfs-http-server/src/api/resources/block.js b/packages/ipfs-http-server/src/api/resources/block.js index be4e47ecae..6c88d85fd6 100644 --- a/packages/ipfs-http-server/src/api/resources/block.js +++ b/packages/ipfs-http-server/src/api/resources/block.js @@ -69,7 +69,7 @@ exports.get = { throw Boom.notFound('Block was unwanted before it could be remotely retrieved') } - return h.response(block.data).header('X-Stream-Output', '1') + return h.response(Buffer.from(block.data.buffer)).header('X-Stream-Output', '1') } } exports.put = { From db4d9d1e2bb62f3c9cde5d67ede9f3e3e31cc9fa Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Thu, 8 Apr 2021 05:54:27 -0700 Subject: [PATCH 3/6] fix(deps): update ipfs-repo to fix types --- packages/ipfs-cli/package.json | 2 +- packages/ipfs-core/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ipfs-cli/package.json b/packages/ipfs-cli/package.json index eefcf8d53f..a6b3874bec 100644 --- a/packages/ipfs-cli/package.json +++ b/packages/ipfs-cli/package.json @@ -42,7 +42,7 @@ "ipfs-core-utils": "^0.7.2", "ipfs-daemon": "^0.5.4", "ipfs-http-client": "^49.0.4", - "ipfs-repo": "github:hannahhoward/js-ipfs-repo#d0ad9a4", + "ipfs-repo": "github:hannahhoward/js-ipfs-repo#dd812a6", "ipfs-utils": "^6.0.4", "ipld-dag-cbor": "^0.18.0", "ipld-dag-pb": "^0.22.1", diff --git a/packages/ipfs-core/package.json b/packages/ipfs-core/package.json index e0d87c166e..e031afc81c 100644 --- a/packages/ipfs-core/package.json +++ b/packages/ipfs-core/package.json @@ -74,7 +74,7 @@ "ipfs-block-service": "^0.19.0", "ipfs-core-types": "^0.3.1", "ipfs-core-utils": "^0.7.2", - "ipfs-repo": "github:hannahhoward/js-ipfs-repo#d0ad9a4", + "ipfs-repo": "github:hannahhoward/js-ipfs-repo#dd812a6", "ipfs-unixfs": "^4.0.1", "ipfs-unixfs-exporter": "^5.0.1", "ipfs-unixfs-importer": "^7.0.1", From 1a96c39bbc461cb2de0f57c870a0c29f18b43f4a Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Thu, 8 Apr 2021 06:10:52 -0700 Subject: [PATCH 4/6] fix(ipfs-core): fix repo gc typing --- packages/ipfs-core/src/components/repo/gc.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/ipfs-core/src/components/repo/gc.js b/packages/ipfs-core/src/components/repo/gc.js index 206091da7e..082f0c2e4b 100644 --- a/packages/ipfs-core/src/components/repo/gc.js +++ b/packages/ipfs-core/src/components/repo/gc.js @@ -17,6 +17,7 @@ const BLOCK_RM_CONCURRENCY = 256 * @typedef {import('ipfs-core-types/src/refs').API} RefsAPI * @typedef {import('ipfs-repo')} IPFSRepo * @typedef {import('interface-datastore').Key} Key + * @typedef {import('ipfs-repo/src/types').Block} Block */ /** @@ -101,7 +102,7 @@ async function createMarkedSet ({ pin, refs, repo }) { * @param {object} arg * @param {IPFSRepo} arg.repo * @param {Set} markedSet - * @param {AsyncIterable} blockKeys + * @param {AsyncIterable} blockKeys */ async function * deleteUnmarkedBlocks ({ repo }, markedSet, blockKeys) { // Iterate through all blocks and find those that are not in the marked set @@ -110,12 +111,16 @@ async function * deleteUnmarkedBlocks ({ repo }, markedSet, blockKeys) { let removedBlocksCount = 0 /** - * @param {CID} cid + * @param {Block|CID} cid */ const removeBlock = async (cid) => { blocksCount++ try { + if (!CID.isCID(cid)) { + return null + } + const b32 = multibase.encode('base32', cid.multihash).toString() if (markedSet.has(b32)) { From 437661807f5faac9f979f0161caaed6a3b8c3429 Mon Sep 17 00:00:00 2001 From: Hannah Howard Date: Thu, 8 Apr 2021 10:29:17 -0700 Subject: [PATCH 5/6] Update packages/ipfs-http-server/src/api/resources/block.js Co-authored-by: Alex Potsides --- packages/ipfs-http-server/src/api/resources/block.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ipfs-http-server/src/api/resources/block.js b/packages/ipfs-http-server/src/api/resources/block.js index 6c88d85fd6..df8108da05 100644 --- a/packages/ipfs-http-server/src/api/resources/block.js +++ b/packages/ipfs-http-server/src/api/resources/block.js @@ -69,7 +69,7 @@ exports.get = { throw Boom.notFound('Block was unwanted before it could be remotely retrieved') } - return h.response(Buffer.from(block.data.buffer)).header('X-Stream-Output', '1') + return h.response(Buffer.from(block.data.buffer, block.data.byteOffset, block.data.byteLength)).header('X-Stream-Output', '1') } } exports.put = { From 178318c6cdb490836199a16cfb2bdf8fe6af067c Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 28 Apr 2021 15:40:27 +0100 Subject: [PATCH 6/6] chore: update types after switch to queryKeys method --- packages/ipfs-core/src/components/repo/gc.js | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/ipfs-core/src/components/repo/gc.js b/packages/ipfs-core/src/components/repo/gc.js index 409899fd5a..e022c23f89 100644 --- a/packages/ipfs-core/src/components/repo/gc.js +++ b/packages/ipfs-core/src/components/repo/gc.js @@ -17,7 +17,7 @@ const BLOCK_RM_CONCURRENCY = 256 * @typedef {import('ipfs-core-types/src/refs').API} RefsAPI * @typedef {import('ipfs-repo')} IPFSRepo * @typedef {import('interface-datastore').Key} Key - * @typedef {import('ipfs-repo/src/types').Block} Block + * @typedef {import('ipld-block')} Block */ /** @@ -101,7 +101,7 @@ async function createMarkedSet ({ pin, refs, repo }) { * @param {object} arg * @param {IPFSRepo} arg.repo * @param {Set} markedSet - * @param {AsyncIterable} blockKeys + * @param {AsyncIterable} blockKeys */ async function * deleteUnmarkedBlocks ({ repo }, markedSet, blockKeys) { // Iterate through all blocks and find those that are not in the marked set @@ -110,16 +110,12 @@ async function * deleteUnmarkedBlocks ({ repo }, markedSet, blockKeys) { let removedBlocksCount = 0 /** - * @param {Block|CID} cid + * @param {CID} cid */ const removeBlock = async (cid) => { blocksCount++ try { - if (!CID.isCID(cid)) { - return null - } - const b32 = multibase.encode('base32', cid.multihash).toString() if (markedSet.has(b32)) {