diff --git a/.travis.yml b/.travis.yml index 8507920f..0aacbeeb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,8 +6,8 @@ stages: - cov node_js: - - '10' - '12' + - '10' os: - linux diff --git a/README.md b/README.md index 7c0059f9..284c303f 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,7 @@ Get a value at the root of the repo. * `key` can be a buffer, a string or a [Key](https://github.com/ipfs/interface-datastore#keys). -[Blocks](https://github.com/ipfs/js-ipfs-block#readme): +[Blocks](https://github.com/ipfs/js-ipld-block#readme): #### `Promise repo.isInitialized ()` @@ -189,13 +189,13 @@ The returned promise resolves to `false` if the repo has not been initialized an #### `Promise repo.blocks.put (block:Block)` -* `block` should be of type [Block](https://github.com/ipfs/js-ipfs-block#readme). +* `block` should be of type [Block](https://github.com/ipfs/js-ipld-block#readme). #### `Promise repo.blocks.putMany (blocks)` Put many blocks. -* `block` should be an Iterable or AsyncIterable that yields entries of type [Block](https://github.com/ipfs/js-ipfs-block#readme). +* `block` should be an Iterable or AsyncIterable that yields entries of type [Block](https://github.com/ipfs/js-ipld-block#readme). #### `Promise repo.blocks.get (cid)` diff --git a/package.json b/package.json index 473e1408..39de6e4b 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ ], "browser": { "rimraf": false, - "datastore-fs": "datastore-level", + "datastore-fs": "datastore-idb", "./src/lock.js": "./src/lock-memory.js", "./src/default-options.js": "./src/default-options-browser.js" }, @@ -43,33 +43,35 @@ "npm": ">=3.0.0" }, "devDependencies": { - "aegir": "^21.4.5", + "aegir": "^21.8.1", "chai": "^4.2.0", "dirty-chai": "^2.0.1", - "lodash": "^4.17.11", + "just-range": "^2.1.0", "memdown": "^5.1.0", "multihashes": "~0.4.15", "multihashing-async": "~0.8.0", "ncp": "^2.0.0", "rimraf": "^3.0.0", - "sinon": "^9.0.1" + "sinon": "^9.0.2" }, "dependencies": { - "base32.js": "~0.1.0", "bignumber.js": "^9.0.0", + "buffer": "^5.5.0", "bytes": "^3.1.0", "cids": "^0.8.0", - "datastore-core": "~0.7.0", + "datastore-core": "^1.0.0", "datastore-fs": "~0.9.0", + "datastore-idb": "^1.0.0", "datastore-level": "~0.14.0", "debug": "^4.1.0", "err-code": "^2.0.0", - "interface-datastore": "^0.8.0", - "ipfs-block": "~0.8.1", - "ipfs-repo-migrations": "~0.1.0", + "interface-datastore": "^0.8.3", + "ipfs-repo-migrations": "^0.2.0", + "ipfs-utils": "^2.2.0", + "ipld-block": "^0.9.1", "just-safe-get": "^2.0.0", "just-safe-set": "^2.1.0", - "lodash.has": "^4.5.2", + "multibase": "^0.7.0", "p-queue": "^6.0.0", "proper-lockfile": "^4.0.0", "sort-keys": "^4.0.0" diff --git a/src/api-addr.js b/src/api-addr.js index 48d005a2..c1a7fc57 100644 --- a/src/api-addr.js +++ b/src/api-addr.js @@ -1,5 +1,6 @@ 'use strict' +const { Buffer } = require('buffer') const Key = require('interface-datastore').Key const apiFile = new Key('api') diff --git a/src/blockstore-utils.js b/src/blockstore-utils.js index b1d00f74..a95e015e 100644 --- a/src/blockstore-utils.js +++ b/src/blockstore-utils.js @@ -1,8 +1,8 @@ 'use strict' -const base32 = require('base32.js') const { Key } = require('interface-datastore') const CID = require('cids') +const multibase = require('multibase') /** * Transform a cid to the appropriate datastore key. @@ -11,8 +11,7 @@ const CID = require('cids') * @returns {Key} */ exports.cidToKey = cid => { - const enc = new base32.Encoder() - return new Key('/' + enc.write(cid.buffer).finalize(), false) + return new Key('/' + multibase.encode('base32', cid.buffer).toString().slice(1).toUpperCase(), false) } /** @@ -22,8 +21,5 @@ exports.cidToKey = cid => { * @returns {CID} */ exports.keyToCid = key => { - // Block key is of the form / - const decoder = new base32.Decoder() - const buff = decoder.write(key.toString().slice(1)).finalize() - return new CID(Buffer.from(buff)) + return new CID(multibase.decode('b' + key.toString().slice(1).toLowerCase())) } diff --git a/src/blockstore.js b/src/blockstore.js index 7d3a4bc0..8cc3675c 100644 --- a/src/blockstore.js +++ b/src/blockstore.js @@ -2,7 +2,7 @@ const core = require('datastore-core') const ShardingStore = core.ShardingDatastore -const Block = require('ipfs-block') +const Block = require('ipld-block') const CID = require('cids') const errcode = require('err-code') const { cidToKey } = require('./blockstore-utils') diff --git a/src/config.js b/src/config.js index ec2982dd..4ad5ec27 100644 --- a/src/config.js +++ b/src/config.js @@ -1,10 +1,10 @@ 'use strict' +const { Buffer } = require('buffer') const Key = require('interface-datastore').Key const { default: Queue } = require('p-queue') const _get = require('just-safe-get') const _set = require('just-safe-set') -const _has = require('lodash.has') const errcode = require('err-code') const errors = require('./errors') @@ -27,7 +27,7 @@ module.exports = (store) => { const encodedValue = await store.get(configKey) const config = JSON.parse(encodedValue.toString()) - if (key !== undefined && !_has(config, key)) { + if (key !== undefined && _get(config, key) === undefined) { throw new errors.NotFoundError(`Key ${key} does not exist in config`) } diff --git a/src/default-options-browser.js b/src/default-options-browser.js index 07e59d1a..8debbd32 100644 --- a/src/default-options-browser.js +++ b/src/default-options-browser.js @@ -4,10 +4,10 @@ module.exports = { lock: 'memory', storageBackends: { - root: require('datastore-level'), - blocks: require('datastore-level'), - keys: require('datastore-level'), - datastore: require('datastore-level') + root: require('datastore-idb'), + blocks: require('datastore-idb'), + keys: require('datastore-idb'), + datastore: require('datastore-idb') }, storageBackendOptions: { root: { diff --git a/src/index.js b/src/index.js index cfe4119d..fcafa703 100644 --- a/src/index.js +++ b/src/index.js @@ -1,13 +1,12 @@ 'use strict' const _get = require('just-safe-get') -const assert = require('assert') -const path = require('path') const debug = require('debug') const Big = require('bignumber.js') const errcode = require('err-code') const migrator = require('ipfs-repo-migrations') const bytes = require('bytes') +const pathJoin = require('ipfs-utils/src/path-join') const constants = require('./constants') const backends = require('./backends') @@ -40,7 +39,9 @@ class IpfsRepo { * @param {object} options - Configuration */ constructor (repoPath, options) { - assert.strictEqual(typeof repoPath, 'string', 'missing repoPath') + if (typeof repoPath !== 'string') { + throw new Error('missing repoPath') + } this.options = buildOptions(options) this.closed = true @@ -112,13 +113,15 @@ class IpfsRepo { this.lockfile = await this._openLock(this.path) log('acquired repo.lock') log('creating datastore') - this.datastore = backends.create('datastore', path.join(this.path, 'datastore'), this.options) + this.datastore = backends.create('datastore', pathJoin(this.path, 'datastore'), this.options) + await this.datastore.open() log('creating blocks') - const blocksBaseStore = backends.create('blocks', path.join(this.path, 'blocks'), this.options) + const blocksBaseStore = backends.create('blocks', pathJoin(this.path, 'blocks'), this.options) + await blocksBaseStore.open() this.blocks = await blockstore(blocksBaseStore, this.options.storageBackendOptions.blocks) log('creating keystore') - this.keys = backends.create('keys', path.join(this.path, 'keys'), this.options) - + this.keys = backends.create('keys', pathJoin(this.path, 'keys'), this.options) + await this.keys.open() const isCompatible = await this.version.check(constants.repoVersion) if (!isCompatible) { if (await this._isAutoMigrationEnabled()) { @@ -152,11 +155,15 @@ class IpfsRepo { */ _getLocker () { if (typeof this.options.lock === 'string') { - assert(lockers[this.options.lock], 'Unknown lock type: ' + this.options.lock) + if (!lockers[this.options.lock]) { + throw new Error('Unknown lock type: ' + this.options.lock) + } return lockers[this.options.lock] } - assert(this.options.lock, 'No lock provided') + if (!this.options.lock) { + throw new Error('No lock provided') + } return this.options.lock } @@ -251,7 +258,13 @@ class IpfsRepo { } } - await Promise.all([this.root, this.blocks, this.keys, this.datastore].map((store) => store.close())) + await Promise.all([ + this.root, + this.blocks, + this.keys, + this.datastore + ].map((store) => store.close())) + log('unlocking') this.closed = true await this._closeLock() diff --git a/src/lock.js b/src/lock.js index 4cd0c5d2..dcc49eed 100644 --- a/src/lock.js +++ b/src/lock.js @@ -1,5 +1,6 @@ 'use strict' +const { LockExistsError } = require('./errors') const path = require('path') const debug = require('debug') const { lock } = require('proper-lockfile') @@ -27,7 +28,16 @@ const STALE_TIME = 20000 exports.lock = async (dir) => { const file = path.join(dir, lockFile) log('locking %s', file) - const release = await lock(dir, { lockfilePath: file, stale: STALE_TIME }) + let release + try { + release = await lock(dir, { lockfilePath: file, stale: STALE_TIME }) + } catch (err) { + if (err.code === 'ELOCKED') { + throw new LockExistsError(`Lock already being held for file: ${file}`) + } else { + throw err + } + } return { close: async () => { // eslint-disable-line require-await release() diff --git a/src/spec.js b/src/spec.js index 881fc5a4..cc839065 100644 --- a/src/spec.js +++ b/src/spec.js @@ -1,5 +1,6 @@ 'use strict' +const { Buffer } = require('buffer') const Key = require('interface-datastore').Key const sortKeys = require('sort-keys') diff --git a/src/version.js b/src/version.js index c7df0960..7a8d6b53 100644 --- a/src/version.js +++ b/src/version.js @@ -1,5 +1,6 @@ 'use strict' +const { Buffer } = require('buffer') const Key = require('interface-datastore').Key const debug = require('debug') const log = debug('repo:version') diff --git a/test/api-addr-test.js b/test/api-addr-test.js index ad7b48b3..4ca856a9 100644 --- a/test/api-addr-test.js +++ b/test/api-addr-test.js @@ -1,6 +1,7 @@ /* eslint-env mocha */ 'use strict' +const { Buffer } = require('buffer') const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect diff --git a/test/blockstore-test.js b/test/blockstore-test.js index 128b4ce8..7e6096b1 100644 --- a/test/blockstore-test.js +++ b/test/blockstore-test.js @@ -2,22 +2,22 @@ /* eslint-env mocha */ 'use strict' +const { Buffer } = require('buffer') const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect const assert = chai.assert -const Block = require('ipfs-block') +const Block = require('ipld-block') const CID = require('cids') -const _ = require('lodash') +const range = require('just-range') const multihashing = require('multihashing-async') -const path = require('path') -const Key = require('interface-datastore').Key -const base32 = require('base32.js') +const tempDir = require('ipfs-utils/src/temp-dir') +const { cidToKey } = require('../src/blockstore-utils') const IPFSRepo = require('../') module.exports = (repo) => { describe('blockstore', () => { - const blockData = _.range(100).map((i) => Buffer.from(`hello-${i}-${Math.random()}`)) + const blockData = range(100).map((i) => Buffer.from(`hello-${i}-${Math.random()}`)) const bData = Buffer.from('hello world') let b @@ -52,8 +52,8 @@ module.exports = (repo) => { it('massive multiwrite', async function () { this.timeout(15000) // add time for ci - const hashes = await Promise.all(_.range(100).map((i) => multihashing(blockData[i], 'sha2-256'))) - await Promise.all(_.range(100).map((i) => { + const hashes = await Promise.all(range(100).map((i) => multihashing(blockData[i], 'sha2-256'))) + await Promise.all(range(100).map((i) => { const block = new Block(blockData[i], new CID(hashes[i])) return repo.blocks.put(block) })) @@ -61,7 +61,7 @@ module.exports = (repo) => { it('.putMany', async function () { this.timeout(15000) // add time for ci - const blocks = await Promise.all(_.range(50).map(async (i) => { + const blocks = await Promise.all(range(50).map(async (i) => { const d = Buffer.from('many' + Math.random()) const hash = await multihashing(d, 'sha2-256') return new Block(d, new CID(hash)) @@ -79,9 +79,12 @@ module.exports = (repo) => { const cid = new CID(hash) let putInvoked = false let commitInvoked = false - otherRepo = new IPFSRepo(path.join(path.basename(repo.path), '/repo-' + Date.now()), { + otherRepo = new IPFSRepo(tempDir(), { storageBackends: { blocks: class ExplodingBlockStore { + open () { + } + close () { } @@ -147,7 +150,7 @@ module.exports = (repo) => { it('massive read', async function () { this.timeout(15000) // add time for ci - await Promise.all(_.range(20 * 100).map(async (i) => { + await Promise.all(range(20 * 100).map(async (i) => { const j = i % blockData.length const hash = await multihashing(blockData[j], 'sha2-256') const block = await repo.blocks.get(new CID(hash)) @@ -210,12 +213,12 @@ module.exports = (repo) => { const data = Buffer.from(`TEST${Date.now()}`) const hash = await multihashing(data, 'sha2-256') const cid = new CID(hash) - const enc = new base32.Encoder() - const key = new Key('/' + enc.write(cid.buffer).finalize(), false) + const key = cidToKey(cid) - otherRepo = new IPFSRepo(path.join(path.basename(repo.path), '/repo-' + Date.now()), { + otherRepo = new IPFSRepo(tempDir(), { storageBackends: { blocks: class ExplodingBlockStore { + open () {} close () { } diff --git a/test/config-test.js b/test/config-test.js index f753de02..285773ad 100644 --- a/test/config-test.js +++ b/test/config-test.js @@ -1,6 +1,7 @@ /* eslint-env mocha */ 'use strict' +const { Buffer } = require('buffer') const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect diff --git a/test/datastore-test.js b/test/datastore-test.js index e77e3f39..a5f5b311 100644 --- a/test/datastore-test.js +++ b/test/datastore-test.js @@ -2,15 +2,16 @@ /* eslint-env mocha */ 'use strict' +const { Buffer } = require('buffer') const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect -const _ = require('lodash') +const range = require('just-range') const Key = require('interface-datastore').Key module.exports = (repo) => { describe('datastore', () => { - const dataList = _.range(100).map((i) => Buffer.from(`hello-${i}-${Math.random()}`)) + const dataList = range(100).map((i) => Buffer.from(`hello-${i}-${Math.random()}`)) const data = Buffer.from('hello world') const b = new Key('hello') @@ -25,7 +26,7 @@ module.exports = (repo) => { it('massive multiwrite', async function () { this.timeout(15000) // add time for ci - await Promise.all(_.range(100).map((i) => { + await Promise.all(range(100).map((i) => { return repo.datastore.put(new Key('hello' + i), dataList[i]) })) }) @@ -39,7 +40,7 @@ module.exports = (repo) => { it('massive read', async function () { this.timeout(15000) // add time for ci - await Promise.all(_.range(20 * 100).map(async (i) => { + await Promise.all(range(20 * 100).map(async (i) => { const j = i % dataList.length const val = await repo.datastore.get(new Key('hello' + j)) expect(val).to.be.eql(dataList[j]) diff --git a/test/is-initialized.js b/test/is-initialized.js index 62843276..ab5cc878 100644 --- a/test/is-initialized.js +++ b/test/is-initialized.js @@ -3,18 +3,16 @@ 'use strict' const chai = require('chai') +const tempDir = require('ipfs-utils/src/temp-dir') chai.use(require('dirty-chai')) const expect = chai.expect -const os = require('os') -const path = require('path') const IPFSRepo = require('../src') describe('isInitialized', () => { let repo beforeEach(() => { - const repoPath = path.join(os.tmpdir(), 'test-repo-for-' + Math.random()) - repo = new IPFSRepo(repoPath) + repo = new IPFSRepo(tempDir(b => 'test-repo-for-' + b)) }) it('should be false before initialization', async () => { diff --git a/test/lock-test.js b/test/lock-test.js index 26df0b09..50e2d194 100644 --- a/test/lock-test.js +++ b/test/lock-test.js @@ -6,6 +6,7 @@ chai.use(require('dirty-chai')) const expect = chai.expect const IPFSRepo = require('../') const lockMemory = require('../src/lock-memory') +const { LockExistsError } = require('./../src/errors') module.exports = (repo) => { describe('Repo lock tests', () => { @@ -17,25 +18,12 @@ module.exports = (repo) => { it('should prevent multiple repos from using the same path', async () => { const repoClone = new IPFSRepo(repo.path, repo.options) - - // Levelup throws an uncaughtException when a lock already exists, catch it - const mochaExceptionHandler = process.listeners('uncaughtException').pop() - process.removeListener('uncaughtException', mochaExceptionHandler) - process.once('uncaughtException', function (err) { - expect(err.message).to.match(/already held|IO error|already being held/) - }) - try { await repoClone.init({}) await repoClone.open() } catch (err) { - if (process.listeners('uncaughtException').length > 0) { - expect(err.message).to.match(/already locked|already held|already being held|ELOCKED/) - } - } finally { - // Reset listeners to maintain test integrity - process.removeAllListeners('uncaughtException') - process.addListener('uncaughtException', mochaExceptionHandler) + expect(err.code) + .to.equal(LockExistsError.code) } }) }) diff --git a/test/node.js b/test/node.js index f443e276..3b515f60 100644 --- a/test/node.js +++ b/test/node.js @@ -7,6 +7,7 @@ const fs = require('fs') const path = require('path') const promisify = require('util').promisify const os = require('os') +const { LockExistsError } = require('../src/errors') const chai = require('chai') chai.use(require('dirty-chai')) @@ -35,7 +36,7 @@ describe('IPFS Repo Tests onNode.js', () => { lock: async (dir) => { const isLocked = await customLock.locked(dir) if (isLocked) { - throw new Error('already locked') + throw new LockExistsError('already locked') } const lockPath = path.join(dir, customLock.lockName) fs.writeFileSync(lockPath, '') diff --git a/test/options-test.js b/test/options-test.js index bec80c49..c5501ea0 100644 --- a/test/options-test.js +++ b/test/options-test.js @@ -4,7 +4,8 @@ const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect -const path = require('path') +const tempDir = require('ipfs-utils/src/temp-dir') +const { isNode } = require('ipfs-utils/src/env') const rimraf = require('rimraf') if (!rimraf.sync) { // browser @@ -13,7 +14,7 @@ if (!rimraf.sync) { const Repo = require('../') describe('custom options tests', () => { - const repoPath = path.join(__dirname, 'slash', 'path') + const repoPath = tempDir() after(() => { rimraf.sync(repoPath) }) @@ -65,9 +66,8 @@ describe('custom options tests', () => { function noop () {} function expectedRepoOptions () { - if (process.browser) { - return require('../src/default-options-browser') + if (isNode) { + return require('../src/default-options') } - - return require('../src/default-options') + return require('../src/default-options-browser') } diff --git a/test/repo-test.js b/test/repo-test.js index c4e53ab0..e80f160b 100644 --- a/test/repo-test.js +++ b/test/repo-test.js @@ -4,10 +4,9 @@ const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect -const path = require('path') +const tempDir = require('ipfs-utils/src/temp-dir') const IPFSRepo = require('../') const Errors = require('../src/errors') -const os = require('os') const bytes = require('bytes') module.exports = (repo) => { @@ -159,7 +158,7 @@ module.exports = (repo) => { count++ } } - const repo = new IPFSRepo(path.join(os.tmpdir(), 'repo-' + Date.now()), { + const repo = new IPFSRepo(tempDir(), { lock: 'memory', storageBackends: { root: FakeDatastore, @@ -186,7 +185,7 @@ module.exports = (repo) => { }) it('should throw non-already-open errors when opening the root', async () => { - const otherRepo = new IPFSRepo(path.join(os.tmpdir(), 'repo-' + Date.now())) + const otherRepo = new IPFSRepo(tempDir()) const err = new Error('wat') otherRepo.root.open = () => { @@ -200,8 +199,8 @@ module.exports = (repo) => { } }) - it('should ingore non-already-open errors when opening the root', async () => { - const otherRepo = new IPFSRepo(path.join(os.tmpdir(), 'repo-' + Date.now())) + it('should ignore non-already-open errors when opening the root', async () => { + const otherRepo = new IPFSRepo(tempDir()) const err = new Error('Already open') let threwError = false @@ -211,7 +210,7 @@ module.exports = (repo) => { throw err } - await otherRepo.init({}) + await otherRepo._openRoot() expect(threwError).to.be.true() }) @@ -235,7 +234,7 @@ module.exports = (repo) => { }) it('should remove the lockfile when opening the repo fails', async () => { - otherRepo = new IPFSRepo(path.join(os.tmpdir(), 'repo-' + Date.now()), { + otherRepo = new IPFSRepo(tempDir(), { storageBackends: { datastore: ExplodingDatastore } @@ -250,7 +249,7 @@ module.exports = (repo) => { }) it('should re-throw the original error even when removing the lockfile fails', async () => { - otherRepo = new IPFSRepo(path.join(os.tmpdir(), 'repo-' + Date.now()), { + otherRepo = new IPFSRepo(tempDir(), { storageBackends: { datastore: ExplodingDatastore } @@ -269,7 +268,7 @@ module.exports = (repo) => { }) it('should throw when repos are not initialised', async () => { - otherRepo = new IPFSRepo(path.join(os.tmpdir(), 'repo-' + Date.now()), { + otherRepo = new IPFSRepo(tempDir(), { storageBackends: { datastore: ExplodingDatastore } @@ -283,7 +282,7 @@ module.exports = (repo) => { }) it('should throw when config is not set', async () => { - otherRepo = new IPFSRepo(path.join(os.tmpdir(), 'repo-' + Date.now())) + otherRepo = new IPFSRepo(tempDir()) otherRepo.config.exists = () => { return false } @@ -304,7 +303,7 @@ module.exports = (repo) => { it('should return the max storage stat when set', async () => { const maxStorage = '1GB' - otherRepo = new IPFSRepo(path.join(os.tmpdir(), 'repo-' + Date.now())) + otherRepo = new IPFSRepo(tempDir()) await otherRepo.init({}) await otherRepo.open() await otherRepo.config.set('Datastore.StorageMax', maxStorage) @@ -316,7 +315,7 @@ module.exports = (repo) => { }) it('should throw unexpected errors when closing', async () => { - otherRepo = new IPFSRepo(path.join(os.tmpdir(), 'repo-' + Date.now())) + otherRepo = new IPFSRepo(tempDir()) await otherRepo.init({}) await otherRepo.open() @@ -335,7 +334,7 @@ module.exports = (repo) => { }) it('should swallow expected errors when closing', async () => { - otherRepo = new IPFSRepo(path.join(os.tmpdir(), 'repo-' + Date.now())) + otherRepo = new IPFSRepo(tempDir()) await otherRepo.init({}) await otherRepo.open() @@ -349,7 +348,7 @@ module.exports = (repo) => { }) it('should throw unexpected errors when checking if the repo has been initialised', async () => { - otherRepo = new IPFSRepo(path.join(os.tmpdir(), 'repo-' + Date.now())) + otherRepo = new IPFSRepo(tempDir()) otherRepo.config.exists = () => { return true diff --git a/test/stat-test.js b/test/stat-test.js index bf565f6c..08209bff 100644 --- a/test/stat-test.js +++ b/test/stat-test.js @@ -4,7 +4,7 @@ const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect -const Block = require('ipfs-block') +const Block = require('ipld-block') const CID = require('cids') module.exports = (repo) => {