From fd22cd861b8081f322446a62e450944f8b4b9e47 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 28 Jul 2020 15:38:20 +0100 Subject: [PATCH 1/2] fix: remove node buffers Updates to the latest changes from interface-datastore and datastore-core Depends on: - [ ] https://github.com/ipfs/interface-datastore/pull/43 - [ ] https://github.com/ipfs/js-datastore-core/pull/27 BREAKING CHANGES: only uses Uint8Arrays internally --- package.json | 16 +++++++--------- src/index.js | 42 ++++++++++++++++++++++++++---------------- test/index.spec.js | 6 ++++-- 3 files changed, 37 insertions(+), 27 deletions(-) diff --git a/package.json b/package.json index 500cc1a..2d5fe80 100644 --- a/package.json +++ b/package.json @@ -5,10 +5,7 @@ "leadMaintainer": "Alex Potsides ", "main": "src/index.js", "scripts": { - "test": "aegir test", - "test:node": "aegir test -t node", - "test:browser": "aegir test -t browser", - "test:webworker": "aegir test -t webworker", + "test": "aegir test -t node", "build": "aegir build", "lint": "aegir lint", "release": "aegir release", @@ -35,19 +32,20 @@ }, "homepage": "https://github.com/ipfs/js-datastore-fs#readme", "dependencies": { - "datastore-core": "^1.1.0", + "datastore-core": "ipfs/js-datastore-core#fix/remove-node-buffers", "fast-write-atomic": "^0.2.0", - "glob": "^7.1.3", - "interface-datastore": "^1.0.2", + "interface-datastore": "ipfs/interface-datastore#fix/remove-node-buffer", + "it-glob": "0.0.8", "mkdirp": "^1.0.4" }, "devDependencies": { - "aegir": "^22.0.0", + "aegir": "^25.0.0", "async-iterator-all": "^1.0.0", "chai": "^4.2.0", - "cids": "~0.8.0", + "cids": "^0.8.3", "detect-node": "^2.0.4", "dirty-chai": "^2.0.1", + "ipfs-utils": "^2.3.1", "memdown": "^5.1.0", "rimraf": "^3.0.2" }, diff --git a/src/index.js b/src/index.js index aea753c..dcf49bf 100644 --- a/src/index.js +++ b/src/index.js @@ -1,7 +1,7 @@ 'use strict' const fs = require('fs') -const glob = require('glob') +const glob = require('it-glob') const mkdirp = require('mkdirp') const promisify = require('util').promisify const writeAtomic = promisify(require('fast-write-atomic')) @@ -117,7 +117,7 @@ class FsDatastore extends Adapter { * Write to the file system without extension. * * @param {Key} key - * @param {Buffer} val + * @param {Uint8Array} val * @returns {Promise} */ async putRaw (key, val) { @@ -131,11 +131,12 @@ class FsDatastore extends Adapter { * Store the given value under the key * * @param {Key} key - * @param {Buffer} val + * @param {Uint8Array} val * @returns {Promise} */ async put (key, val) { const parts = this._encode(key) + try { await mkdirp(parts.dir, { fs: fs }) await writeFile(parts.file, val) @@ -148,7 +149,7 @@ class FsDatastore extends Adapter { * Read from the file system without extension. * * @param {Key} key - * @returns {Promise} + * @returns {Promise} */ async getRaw (key) { const parts = this._encode(key) @@ -167,7 +168,7 @@ class FsDatastore extends Adapter { * Read from the file system. * * @param {Key} key - * @returns {Promise} + * @returns {Promise} */ async get (key) { const parts = this._encode(key) @@ -216,22 +217,31 @@ class FsDatastore extends Adapter { } async * _all (q) { // eslint-disable-line require-await - // glob expects a POSIX path - const prefix = q.prefix || '**' - const pattern = path - .join(this.path, prefix, '*' + this.opts.extension) + let prefix = q.prefix || '**' + + // strip leading slashes + prefix = prefix.replace(/^\/+/, '') + + const pattern = `${prefix}/*${this.opts.extension}` .split(path.sep) .join('/') - const files = glob.sync(pattern) + const files = glob(this.path, pattern, { + absolute: true + }) if (!q.keysOnly) { - yield * map(files, async (f) => { - const buf = await fsReadFile(f) - return { - key: this._decode(f), - value: buf + for await (const file of files) { + try { + const buf = await fsReadFile(file) + + yield { + key: this._decode(file), + value: buf + } + } catch (err) { + console.info(err) } - }) + } } else { yield * map(files, f => ({ key: this._decode(f) })) } diff --git a/test/index.spec.js b/test/index.spec.js index 7de991c..8e18a30 100644 --- a/test/index.spec.js +++ b/test/index.spec.js @@ -16,6 +16,8 @@ const utils = require('interface-datastore').utils const ShardingStore = require('datastore-core').ShardingDatastore const sh = require('datastore-core').shard const isNode = require('detect-node') +const TextEncoder = require('ipfs-utils/src/text-encoder') +const utf8Encoder = new TextEncoder('utf8') const FsStore = require('../src') @@ -87,7 +89,7 @@ describe('FsDatastore', () => { const fs = new FsStore(dir) const key = new Key('1234') - await fs.put(key, Buffer.from([0, 1, 2, 3])) + await fs.put(key, Uint8Array.from([0, 1, 2, 3])) await fs.delete(key) try { @@ -181,7 +183,7 @@ describe('FsDatastore', () => { const dir = utils.tmpdir() const fstore = new FsStore(dir) const key = new Key('CIQGFTQ7FSI2COUXWWLOQ45VUM2GUZCGAXLWCTOKKPGTUWPXHBNIVOY') - const value = Buffer.from('Hello world') + const value = utf8Encoder.encode('Hello world') await Promise.all( new Array(100).fill(0).map(() => fstore.put(key, value)) From b36b0f285e39ee6cc7b669dc46736c5e5c1626c7 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 29 Jul 2020 13:48:10 +0100 Subject: [PATCH 2/2] chore: remove gh deps and do not swallow error --- package.json | 4 ++-- src/index.js | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 2d5fe80..a8f0f8c 100644 --- a/package.json +++ b/package.json @@ -32,9 +32,9 @@ }, "homepage": "https://github.com/ipfs/js-datastore-fs#readme", "dependencies": { - "datastore-core": "ipfs/js-datastore-core#fix/remove-node-buffers", + "datastore-core": "^2.0.0", "fast-write-atomic": "^0.2.0", - "interface-datastore": "ipfs/interface-datastore#fix/remove-node-buffer", + "interface-datastore": "^2.0.0", "it-glob": "0.0.8", "mkdirp": "^1.0.4" }, diff --git a/src/index.js b/src/index.js index dcf49bf..6644ea1 100644 --- a/src/index.js +++ b/src/index.js @@ -239,7 +239,11 @@ class FsDatastore extends Adapter { value: buf } } catch (err) { - console.info(err) + // if keys are removed from the datastore while the query is + // running, we may encounter missing files. + if (err.code !== 'ENOENT') { + throw err + } } } } else {