Skip to content
This repository was archived by the owner on Mar 23, 2023. It is now read-only.

fix: remove node buffers #44

Merged
merged 2 commits into from
Jul 29, 2020
Merged

fix: remove node buffers #44

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fd22cd8
fix: remove node buffers
achingbrain Jul 28, 2020
b36b0f2
chore: remove gh deps and do not swallow error
achingbrain Jul 29, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 7 additions & 9 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,7 @@
"leadMaintainer": "Alex Potsides <alex.potsides@protocol.ai>",
"main": "src/index.js",
"scripts": {
"test": "aegir test",
"test:node": "aegir test -t node",
"test:browser": "aegir test -t browser",
"test:webworker": "aegir test -t webworker",
"test": "aegir test -t node",
"build": "aegir build",
"lint": "aegir lint",
"release": "aegir release",
Expand All @@ -35,19 +32,20 @@
},
"homepage": "https://github.com/ipfs/js-datastore-fs#readme",
"dependencies": {
"datastore-core": "^1.1.0",
"datastore-core": "ipfs/js-datastore-core#fix/remove-node-buffers",
"fast-write-atomic": "^0.2.0",
"glob": "^7.1.3",
"interface-datastore": "^1.0.2",
"interface-datastore": "ipfs/interface-datastore#fix/remove-node-buffer",
"it-glob": "0.0.8",
"mkdirp": "^1.0.4"
},
"devDependencies": {
"aegir": "^22.0.0",
"aegir": "^25.0.0",
"async-iterator-all": "^1.0.0",
"chai": "^4.2.0",
"cids": "~0.8.0",
"cids": "^0.8.3",
"detect-node": "^2.0.4",
"dirty-chai": "^2.0.1",
"ipfs-utils": "^2.3.1",
"memdown": "^5.1.0",
"rimraf": "^3.0.2"
},
Expand Down
42 changes: 26 additions & 16 deletions src/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
'use strict'

const fs = require('fs')
const glob = require('glob')
const glob = require('it-glob')
const mkdirp = require('mkdirp')
const promisify = require('util').promisify
const writeAtomic = promisify(require('fast-write-atomic'))
Expand Down Expand Up @@ -117,7 +117,7 @@ class FsDatastore extends Adapter {
* Write to the file system without extension.
*
* @param {Key} key
* @param {Buffer} val
* @param {Uint8Array} val
* @returns {Promise<void>}
*/
async putRaw (key, val) {
Expand All @@ -131,11 +131,12 @@ class FsDatastore extends Adapter {
* Store the given value under the key
*
* @param {Key} key
* @param {Buffer} val
* @param {Uint8Array} val
* @returns {Promise<void>}
*/
async put (key, val) {
const parts = this._encode(key)

try {
await mkdirp(parts.dir, { fs: fs })
await writeFile(parts.file, val)
Expand All @@ -148,7 +149,7 @@ class FsDatastore extends Adapter {
* Read from the file system without extension.
*
* @param {Key} key
* @returns {Promise<Buffer>}
* @returns {Promise<Uint8Array>}
*/
async getRaw (key) {
const parts = this._encode(key)
Expand All @@ -167,7 +168,7 @@ class FsDatastore extends Adapter {
* Read from the file system.
*
* @param {Key} key
* @returns {Promise<Buffer>}
* @returns {Promise<Uint8Array>}
*/
async get (key) {
const parts = this._encode(key)
Expand Down Expand Up @@ -216,22 +217,31 @@ class FsDatastore extends Adapter {
}

async * _all (q) { // eslint-disable-line require-await
// glob expects a POSIX path
const prefix = q.prefix || '**'
const pattern = path
.join(this.path, prefix, '*' + this.opts.extension)
let prefix = q.prefix || '**'

// strip leading slashes
prefix = prefix.replace(/^\/+/, '')

const pattern = `${prefix}/*${this.opts.extension}`
.split(path.sep)
.join('/')
const files = glob.sync(pattern)
const files = glob(this.path, pattern, {
absolute: true
})

if (!q.keysOnly) {
yield * map(files, async (f) => {
const buf = await fsReadFile(f)
return {
key: this._decode(f),
value: buf
for await (const file of files) {
try {
const buf = await fsReadFile(file)

yield {
key: this._decode(file),
value: buf
}
} catch (err) {
console.info(err)
Copy link

@Gozala Gozala Jul 28, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚨 It looks like this would start swallowing errors, which seems unrelated to Uint8Array change.
📝 Should this be console.error(error) (or console.warn) instead ?

Copy link
Member Author

@achingbrain achingbrain Jul 28, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oops, no, this shouldn't be in there at all. It's related to a new test added to interface-datastore that mutates the datastore while running a query (a value is deleted). To pass the test we don't have to take the mutation into account but we shouldn't explode either.

}
})
}
} else {
yield * map(files, f => ({ key: this._decode(f) }))
}
Expand Down
6 changes: 4 additions & 2 deletions test/index.spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ const utils = require('interface-datastore').utils
const ShardingStore = require('datastore-core').ShardingDatastore
const sh = require('datastore-core').shard
const isNode = require('detect-node')
const TextEncoder = require('ipfs-utils/src/text-encoder')
const utf8Encoder = new TextEncoder('utf8')

const FsStore = require('../src')

Expand Down Expand Up @@ -87,7 +89,7 @@ describe('FsDatastore', () => {
const fs = new FsStore(dir)
const key = new Key('1234')

await fs.put(key, Buffer.from([0, 1, 2, 3]))
await fs.put(key, Uint8Array.from([0, 1, 2, 3]))
await fs.delete(key)

try {
Expand Down Expand Up @@ -181,7 +183,7 @@ describe('FsDatastore', () => {
const dir = utils.tmpdir()
const fstore = new FsStore(dir)
const key = new Key('CIQGFTQ7FSI2COUXWWLOQ45VUM2GUZCGAXLWCTOKKPGTUWPXHBNIVOY')
const value = Buffer.from('Hello world')
const value = utf8Encoder.encode('Hello world')

await Promise.all(
new Array(100).fill(0).map(() => fstore.put(key, value))
Expand Down