diff --git a/.gitignore b/.gitignore index 9a3cb9fa..7ad9e674 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,9 @@ node_modules +build dist .docs .coverage node_modules package-lock.json yarn.lock +.vscode diff --git a/packages/interface-blockstore-tests/package.json b/packages/interface-blockstore-tests/package.json index 9da31e84..fa5dcd94 100644 --- a/packages/interface-blockstore-tests/package.json +++ b/packages/interface-blockstore-tests/package.json @@ -23,22 +23,6 @@ }, "type": "module", "types": "./dist/src/index.d.ts", - "typesVersions": { - "*": { - "*": [ - "*", - "dist/*", - "dist/src/*", - "dist/src/*/index" - ], - "src/*": [ - "*", - "dist/*", - "dist/src/*", - "dist/src/*/index" - ] - } - }, "files": [ "src", "dist", @@ -154,7 +138,6 @@ "interface-blockstore": "^4.0.0", "it-all": "^2.0.0", "it-drain": "^2.0.0", - "it-length": "^2.0.0", "multiformats": "^11.0.0", "uint8arrays": "^4.0.2" }, diff --git a/packages/interface-blockstore-tests/src/index.ts b/packages/interface-blockstore-tests/src/index.ts index 738e8a2c..dddae577 100644 --- a/packages/interface-blockstore-tests/src/index.ts +++ b/packages/interface-blockstore-tests/src/index.ts @@ -4,19 +4,18 @@ import { expect } from 'aegir/chai' import all from 'it-all' import drain from 'it-drain' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -import { CID } from 'multiformats/cid' import { sha256 } from 'multiformats/hashes/sha2' +import type { Blockstore, Pair } from 'interface-blockstore' import { base32 } from 'multiformats/bases/base32' +import { CID } from 'multiformats/cid' import * as raw from 'multiformats/codecs/raw' -import length from 'it-length' -import type { Blockstore, KeyQueryFilter, KeyQueryOrder, Pair, QueryFilter, QueryOrder } from 'interface-blockstore' async function getKeyValuePair (data?: string): Promise { - const value = uint8ArrayFromString(data ?? `data-${Math.random()}`) - const hash = await sha256.digest(value) - const key = CID.createV1(raw.code, hash) + const block = uint8ArrayFromString(data ?? `data-${Math.random()}`) + const multihash = await sha256.digest(block) + const cid = CID.createV1(raw.code, multihash) - return { key, value } + return { cid, block } } async function getKeyValuePairs (count: number): Promise { @@ -25,103 +24,110 @@ async function getKeyValuePairs (count: number): Promise { ) } -export function interfaceBlockstoreTests (test: { teardown: () => void, setup: () => Blockstore }): void { - const cleanup = async (store: Blockstore): Promise => { - await store.close() - test.teardown() +export interface InterfaceBlockstoreTest { + setup: () => B | Promise + teardown: (store: B) => void | Promise +} + +export function interfaceBlockstoreTests (test: InterfaceBlockstoreTest): void { + const cleanup = async (store: B): Promise => { + await test.teardown(store) } - const createStore = async (): Promise => { - const store = test.setup() + const createStore = async (): Promise => { + const store = await test.setup() + if (store == null) { throw new Error('missing store') } - await store.open() + return store } describe('put', () => { - let store: Blockstore + let store: B beforeEach(async () => { store = await createStore() - await store.open() }) afterEach(async () => { - await store.close() await cleanup(store) }) it('simple', async () => { - const { key, value } = await getKeyValuePair() + const { cid, block } = await getKeyValuePair() - await store.put(key, value) + await store.put(cid, block) }) it('parallel', async () => { const data = await getKeyValuePairs(100) - await Promise.all(data.map(async d => { await store.put(d.key, d.value) })) + await Promise.all(data.map(async d => { await store.put(d.cid, d.block) })) - const res = await all(store.getMany(data.map(d => d.key))) - expect(res).to.deep.equal(data.map(d => d.value)) + const res = await all(store.getMany(data.map(d => d.cid))) + expect(res).to.deep.equal(data.map(d => d.block)) }) }) describe('putMany', () => { - let store: Blockstore + let store: B beforeEach(async () => { store = await createStore() }) - afterEach(async () => { await cleanup(store) }) + afterEach(async () => { + await cleanup(store) + }) it('streaming', async () => { const data = await getKeyValuePairs(100) let index = 0 - for await (const { key, value } of store.putMany(data)) { - expect(data[index]).to.deep.equal({ key, value }) + for await (const { cid, block } of store.putMany(data)) { + expect(data[index]).to.deep.equal({ cid, block }) index++ } expect(index).to.equal(data.length) - const res = await all(store.getMany(data.map(d => d.key))) - expect(res).to.deep.equal(data.map(d => d.value)) + const res = await all(store.getMany(data.map(d => d.cid))) + expect(res).to.deep.equal(data.map(d => d.block)) }) }) describe('get', () => { - let store: Blockstore + let store: B beforeEach(async () => { store = await createStore() }) - afterEach(async () => { await cleanup(store) }) + afterEach(async () => { + await cleanup(store) + }) it('simple', async () => { const { - key, value + cid, block } = await getKeyValuePair() - await store.put(key, value) + await store.put(cid, block) - const res = await store.get(key) - expect(res).to.equalBytes(value) + const res = await store.get(cid) + expect(res).to.equalBytes(block) }) it('should throw error for missing key', async () => { const { - key + cid } = await getKeyValuePair() try { - await store.get(key) + await store.get(cid) } catch (err) { expect(err).to.have.property('code', 'ERR_NOT_FOUND') return @@ -132,34 +138,36 @@ export function interfaceBlockstoreTests (test: { teardown: () => void, setup: ( }) describe('getMany', () => { - let store: Blockstore + let store: B beforeEach(async () => { store = await createStore() }) - afterEach(async () => { await cleanup(store) }) + afterEach(async () => { + await cleanup(store) + }) it('streaming', async () => { const { - key, value + cid, block } = await getKeyValuePair() - await store.put(key, value) - const source = [key] + await store.put(cid, block) + const source = [cid] const res = await all(store.getMany(source)) expect(res).to.have.lengthOf(1) - expect(res[0]).to.equalBytes(value) + expect(res[0]).to.equalBytes(block) }) it('should throw error for missing key', async () => { const { - key + cid } = await getKeyValuePair() try { - await drain(store.getMany([key])) + await drain(store.getMany([cid])) } catch (err) { expect(err).to.have.property('code', 'ERR_NOT_FOUND') return @@ -170,415 +178,110 @@ export function interfaceBlockstoreTests (test: { teardown: () => void, setup: ( }) describe('delete', () => { - let store: Blockstore + let store: B beforeEach(async () => { store = await createStore() }) - afterEach(async () => { await cleanup(store) }) + afterEach(async () => { + await cleanup(store) + }) it('simple', async () => { const { - key, value + cid, block } = await getKeyValuePair() - await store.put(key, value) - await store.get(key) - await store.delete(key) - const exists = await store.has(key) + await store.put(cid, block) + await store.get(cid) + await store.delete(cid) + const exists = await store.has(cid) expect(exists).to.be.eql(false) }) it('parallel', async () => { const data = await getKeyValuePairs(100) - await Promise.all(data.map(async d => { await store.put(d.key, d.value) })) + await Promise.all(data.map(async d => { await store.put(d.cid, d.block) })) - const res0 = await Promise.all(data.map(async d => await store.has(d.key))) + const res0 = await Promise.all(data.map(async d => await store.has(d.cid))) res0.forEach(res => expect(res).to.be.eql(true)) - await Promise.all(data.map(async d => { await store.delete(d.key) })) + await Promise.all(data.map(async d => { await store.delete(d.cid) })) - const res1 = await Promise.all(data.map(async d => await store.has(d.key))) + const res1 = await Promise.all(data.map(async d => await store.has(d.cid))) res1.forEach(res => expect(res).to.be.eql(false)) }) }) describe('deleteMany', () => { - let store: Blockstore + let store: B beforeEach(async () => { store = await createStore() }) - afterEach(async () => { await cleanup(store) }) + afterEach(async () => { + await cleanup(store) + }) it('streaming', async () => { const data = await getKeyValuePairs(100) await drain(store.putMany(data)) - const res0 = await Promise.all(data.map(async d => await store.has(d.key))) + const res0 = await Promise.all(data.map(async d => await store.has(d.cid))) res0.forEach(res => expect(res).to.be.eql(true)) let index = 0 - for await (const key of store.deleteMany(data.map(d => d.key))) { - expect(data[index].key).to.deep.equal(key) + for await (const key of store.deleteMany(data.map(d => d.cid))) { + expect(data[index].cid).to.deep.equal(key) index++ } expect(index).to.equal(data.length) - const res1 = await Promise.all(data.map(async d => await store.has(d.key))) + const res1 = await Promise.all(data.map(async d => await store.has(d.cid))) res1.forEach(res => expect(res).to.be.eql(false)) }) }) - describe('batch', () => { - let store: Blockstore + describe('blocks', () => { + let store: B beforeEach(async () => { store = await createStore() }) - afterEach(async () => { await cleanup(store) }) - - it('simple', async () => { - const data = await getKeyValuePairs(4) - - const b = store.batch() - - await store.put(data[0].key, data[0].value) - - b.put(data[1].key, data[1].value) - b.put(data[2].key, data[2].value) - b.put(data[3].key, data[3].value) - b.delete(data[0].key) - await b.commit() - - const keys = data.map(d => d.key) - const res = await Promise.all(keys.map(async k => await store.has(k))) - - expect(res).to.be.eql([false, true, true, true]) - }) - - it('many (100)', async function () { - this.timeout(320 * 1000) - const b = store.batch() - const count = 100 - - const prefixes: Record = {} - - for (let i = 0; i < count; i++) { - const { - key, value - } = await getKeyValuePair() - - b.put(key, value) - - // find the shortest stringified key that aligns with a byte boundary - const keyStr = key.toString() - let prefix = '' - - for (let j = keyStr.length - 1; j > 20; j--) { - try { - base32.decode(keyStr.substring(0, j)) - prefix = keyStr.substring(0, j) - } catch (err: any) { - if (err.message !== 'Unexpected end of data') { - throw err - } - } - } - - prefixes[prefix] = (prefixes[prefix] ?? 0) + 1 - } - - await b.commit() - - await Promise.all( - Object.keys(prefixes) - .map(async prefix => { - await expect(length(store.query({ prefix }))).to.eventually.equal(prefixes[prefix]) - }) - ) - }) - }) - - describe('query', () => { - let store: Blockstore - let hello: Pair - let world: Pair - let hello2: Pair - let filter1: QueryFilter - let filter2: QueryFilter - - before(async () => { - hello = await getKeyValuePair('hello') - hello2 = await getKeyValuePair('hello2') - world = await getKeyValuePair('world') - - filter1 = entry => !entry.key.toString().endsWith(hello.key.toString().substring(-5)) - filter2 = entry => entry.key.toString().endsWith(hello2.key.toString().substring(-5)) - }) - - const order1: QueryOrder = (a, b) => { - if (a.value.toString() < b.value.toString()) { - return -1 - } - return 1 - } - - const order2: QueryOrder = (a, b) => { - if (a.value.toString() < b.value.toString()) { - return 1 - } - if (a.value.toString() > b.value.toString()) { - return -1 - } - return 0 - } - - const tests: Array<{ name: string, test: () => { query: any, expected: any } }> = [ - { name: 'empty', test: () => ({ query: {}, expected: [hello, world, hello2] }) }, - { name: 'prefix', test: () => ({ query: { prefix: hello.key.toString().charAt(0) }, expected: [hello, world, hello2] }) }, - { name: '1 filter', test: () => ({ query: { filters: [filter1] }, expected: [world, hello2] }) }, - { name: '2 filters', test: () => ({ query: { filters: [filter1, filter2] }, expected: [hello2] }) }, - { name: 'limit', test: () => ({ query: { limit: 1 }, expected: 1 }) }, - { name: 'offset', test: () => ({ query: { offset: 1 }, expected: 2 }) }, - { name: '1 order (1)', test: () => ({ query: { orders: [order1] }, expected: [hello, hello2, world] }) }, - { name: '1 order (reverse 1)', test: () => ({ query: { orders: [order2] }, expected: [world, hello2, hello] }) } - ] - - before(async () => { - store = await createStore() - - const b = store.batch() - - b.put(hello.key, hello.value) - b.put(world.key, world.value) - b.put(hello2.key, hello2.value) - - await b.commit() - }) - - after(async () => { await cleanup(store) }) - - tests.forEach(({ name, test }) => it(name, async () => { - const { - query, expected - } = test() - let res = await all(store.query(query)) - - if (Array.isArray(expected)) { - if (query.orders == null) { - expect(res).to.have.length(expected.length) - const s = (a: Pair, b: Pair): number => { - if (a.key.toString() < b.key.toString()) { - return 1 - } else { - return -1 - } - } - res = res.sort(s) - const exp = expected.sort(s) - - res.forEach((r, i) => { - expect(r.key.toString()).to.be.eql(exp[i].key.toString()) - - if (r.value == null) { - expect(exp[i].value).to.not.exist() - } else { - expect(r.value).to.deep.equal(exp[i].value) - } - }) - } else { - expect(res).to.be.eql(expected) - } - } else if (typeof expected === 'number') { - expect(res).to.have.length(expected) - } - })) - - it('allows mutating the datastore during a query', async () => { - const hello3 = await getKeyValuePair() - let firstIteration = true - - for await (const {} of store.query({})) { // eslint-disable-line no-empty-pattern - if (firstIteration) { - expect(await store.has(hello2.key)).to.be.true() - await store.delete(hello2.key) - expect(await store.has(hello2.key)).to.be.false() - - await store.put(hello3.key, hello3.value) - firstIteration = false - } - } - - const results = await all(store.query({})) - - expect(firstIteration).to.be.false('Query did not return anything') - expect(results.map(result => result.key)).to.have.deep.members([ - hello.key, - world.key, - hello3.key - ]) - }) - - it('queries while the datastore is being mutated', async () => { - const { - key, value - } = await getKeyValuePair() - const writePromise = store.put(key, value) - const results = await all(store.query({})) - expect(results.length).to.be.greaterThan(0) - await writePromise - }) - }) - - describe('queryKeys', () => { - let store: Blockstore - let hello: Pair - let world: Pair - let hello2: Pair - let filter1: KeyQueryFilter - let filter2: KeyQueryFilter - - before(async () => { - hello = await getKeyValuePair('hello') - hello2 = await getKeyValuePair('hello2') - world = await getKeyValuePair('world') - - filter1 = key => !key.toString().endsWith(hello.key.toString().substring(-5)) - filter2 = key => key.toString().endsWith(hello2.key.toString().substring(-5)) + afterEach(async () => { + await cleanup(store) }) - const order1: KeyQueryOrder = (a, b) => { - if (a.toString() < b.toString()) { - return -1 - } - return 1 - } - - const order2: KeyQueryOrder = (a, b) => { - if (a.toString() < b.toString()) { - return 1 - } - if (a.toString() > b.toString()) { - return -1 - } - return 0 - } + it('returns all blocks', async () => { + const data = await getKeyValuePairs(100) - const tests: Array<{ name: string, test: () => { query: any, expected: any } }> = [ - { name: 'empty', test: () => ({ query: {}, expected: [hello.key, world.key, hello2.key] }) }, - { name: 'prefix', test: () => ({ query: { prefix: hello.key.toString().charAt(0) }, expected: [hello.key, world.key, hello2.key] }) }, - { name: '1 filter', test: () => ({ query: { filters: [filter1] }, expected: [world.key, hello2.key] }) }, - { name: '2 filters', test: () => ({ query: { filters: [filter1, filter2] }, expected: [hello2.key] }) }, - { name: 'limit', test: () => ({ query: { limit: 1 }, expected: 1 }) }, - { name: 'offset', test: () => ({ query: { offset: 1 }, expected: 2 }) }, - { name: '1 order (1)', test: () => ({ query: { orders: [order1] }, expected: [hello.key, world.key, hello2.key] }) }, - { name: '1 order (reverse 1)', test: () => ({ query: { orders: [order2] }, expected: [hello2.key, world.key, hello.key] }) } - ] - - before(async () => { - store = await createStore() + await drain(store.putMany(data)) - const b = store.batch() + const allBlocks = await all(store.getAll()) - b.put(hello.key, hello.value) - b.put(world.key, world.value) - b.put(hello2.key, hello2.value) + expect(allBlocks).of.have.lengthOf(data.length) - await b.commit() - }) + // order is not preserved + for (const { cid, block } of data) { + const retrievedPair = allBlocks.find(pair => { + return base32.encode(cid.multihash.bytes) === base32.encode(pair.cid.multihash.bytes) + }) - after(async () => { await cleanup(store) }) + expect(retrievedPair).to.be.ok() - tests.forEach(({ name, test }) => it(name, async () => { - const { - query, expected - } = test() - let res = await all(store.queryKeys(query)) - - if (Array.isArray(expected)) { - if (query.orders == null) { - expect(res).to.have.length(expected.length) - - const s: KeyQueryOrder = (a, b) => { - if (a.toString() < b.toString()) { - return 1 - } else { - return -1 - } - } - res = res.sort(s) - const exp = expected.sort(s) - - res.forEach((r, i) => { - expect(r.toString()).to.be.eql(exp[i].toString()) - }) - } else { - expect(res).to.be.eql(expected) + if (retrievedPair == null) { + throw new Error('Could not find cid/block pair') } - } else if (typeof expected === 'number') { - expect(res).to.have.length(expected) - } - })) - it('allows mutating the datastore during a query', async () => { - const hello3 = await getKeyValuePair() - let firstIteration = true - - for await (const {} of store.queryKeys({})) { // eslint-disable-line no-empty-pattern - if (firstIteration) { - expect(await store.has(hello2.key)).to.be.true() - await store.delete(hello2.key) - expect(await store.has(hello2.key)).to.be.false() - - await store.put(hello3.key, hello3.value) - firstIteration = false - } + expect(retrievedPair.block).to.equalBytes(block) } - - const results = await all(store.queryKeys({})) - - expect(firstIteration).to.be.false('Query did not return anything') - expect(results).to.have.deep.members([ - hello.key, - world.key, - hello3.key - ]) - }) - - it('queries while the datastore is being mutated', async () => { - const { key, value } = await getKeyValuePair() - const writePromise = store.put(key, value) - const results = await all(store.queryKeys({})) - expect(results.length).to.be.greaterThan(0) - await writePromise - }) - }) - - describe('lifecycle', () => { - let store: Blockstore - - before(() => { - store = test.setup() - if (store == null) { - throw new Error('missing store') - } - }) - - after(async () => { await cleanup(store) }) - - it('close and open', async () => { - await store.close() - await store.open() - await store.close() - await store.open() }) }) } diff --git a/packages/interface-blockstore/src/index.ts b/packages/interface-blockstore/src/index.ts index 8885d5d0..861f0ada 100644 --- a/packages/interface-blockstore/src/index.ts +++ b/packages/interface-blockstore/src/index.ts @@ -1,55 +1,30 @@ import type { - Pair as StorePair, - Batch as StoreBatch, - QueryFilter as StoreQueryFilter, - QueryOrder as StoreQueryOrder, - Query as StoreQuery, - KeyQueryFilter as StoreKeyQueryFilter, - KeyQueryOrder as StoreKeyQueryOrder, - KeyQuery as StoreKeyQuery, Options as StoreOptions, + AwaitIterable, Store } from 'interface-store' -import type { - CID -} from 'multiformats' +import type { CID } from 'multiformats/cid' export interface Options extends StoreOptions { } -export interface Pair extends StorePair { - -} - -export interface Batch extends StoreBatch { - -} - -export interface Blockstore extends Store { - -} - -export interface QueryFilter extends StoreQueryFilter { - -} - -export interface QueryOrder extends StoreQueryOrder { - -} - -export interface Query extends StoreQuery { - -} - -export interface KeyQueryFilter extends StoreKeyQueryFilter { - -} - -export interface KeyQueryOrder extends StoreKeyQueryOrder { - +export interface Pair { + cid: CID + block: Uint8Array } -export interface KeyQuery extends StoreKeyQuery { - +export interface Blockstore extends Store { + /** + * Retrieve all cid/block pairs from the blockstore as an unordered iterable + * + * @example + * ```js + * for await (const { multihash, block } of store.getAll()) { + * console.log('got:', multihash, block) + * // => got MultihashDigest('Qmfoo') Uint8Array[...] + * } + * ``` + */ + getAll: (options?: Options) => AwaitIterable } diff --git a/packages/interface-datastore-tests/package.json b/packages/interface-datastore-tests/package.json index ea2f1c12..8cad1e34 100644 --- a/packages/interface-datastore-tests/package.json +++ b/packages/interface-datastore-tests/package.json @@ -23,22 +23,6 @@ }, "type": "module", "types": "./dist/src/index.d.ts", - "typesVersions": { - "*": { - "*": [ - "*", - "dist/*", - "dist/src/*", - "dist/src/*/index" - ], - "src/*": [ - "*", - "dist/*", - "dist/src/*", - "dist/src/*/index" - ] - } - }, "files": [ "src", "dist", @@ -48,7 +32,7 @@ "exports": { ".": { "types": "./dist/src/index.d.ts", - "import": "./src/index.js" + "import": "./dist/src/index.js" } }, "eslintConfig": { @@ -155,9 +139,10 @@ "iso-random-stream": "^2.0.0", "it-all": "^2.0.0", "it-drain": "^2.0.0", + "it-length": "^2.0.1", "uint8arrays": "^4.0.2" }, "typedoc": { - "entryPoint": "./src/index.js" + "entryPoint": "./src/index.ts" } } diff --git a/packages/interface-datastore-tests/src/index.js b/packages/interface-datastore-tests/src/index.ts similarity index 72% rename from packages/interface-datastore-tests/src/index.js rename to packages/interface-datastore-tests/src/index.ts index 03fd5544..79fb20bd 100644 --- a/packages/interface-datastore-tests/src/index.js +++ b/packages/interface-datastore-tests/src/index.ts @@ -5,49 +5,35 @@ import { expect } from 'aegir/chai' import all from 'it-all' import drain from 'it-drain' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' -import { Key } from 'interface-datastore' - -/** - * @typedef {import('interface-datastore').Datastore} Datastore - * @typedef {import('interface-datastore').Pair} Pair - * @typedef {import('interface-datastore').QueryOrder} QueryOrder - * @typedef {import('interface-datastore').QueryFilter} QueryFilter - * @typedef {import('interface-datastore').KeyQueryOrder} KeyQueryOrder - * @typedef {import('interface-datastore').KeyQueryFilter} KeyQueryFilter - */ - -/** - * @param {{ teardown: () => void; setup: () => Datastore; }} test - */ -export function interfaceDatastoreTests (test) { - /** - * @param {Datastore} store - */ - const cleanup = async store => { - await store.close() - await test.teardown() +import { Datastore, Key, KeyQueryFilter, KeyQueryOrder, QueryFilter, QueryOrder } from 'interface-datastore' +import length from 'it-length' + +export interface InterfacDatastoreTest { + setup: () => D | Promise + teardown: (store: D) => void | Promise +} + +export function interfaceDatastoreTests (test: InterfacDatastoreTest): void { + const cleanup = async (store: D): Promise => { + await test.teardown(store) } - const createStore = async () => { - const store = await test.setup() - if (!store) throw new Error('missing store') - await store.open() - return store + const createStore = async (): Promise => { + return await test.setup() } describe('put', () => { - /** @type {Datastore} */ - let store + let store: D beforeEach(async () => { store = await createStore() }) - afterEach(() => cleanup(store)) + afterEach(async () => { await cleanup(store) }) - it('simple', () => { + it('simple', async () => { const k = new Key('/z/one') - return store.put(k, uint8ArrayFromString('one')) + await store.put(k, uint8ArrayFromString('one')) }) it('parallel', async () => { @@ -56,7 +42,7 @@ export function interfaceDatastoreTests (test) { data.push({ key: new Key(`/z/key${i}`), value: uint8ArrayFromString(`data${i}`) }) } - await Promise.all(data.map(d => store.put(d.key, d.value))) + await Promise.all(data.map(async d => { await store.put(d.key, d.value) })) const res = await all(store.getMany(data.map(d => d.key))) expect(res).to.deep.equal(data.map(d => d.value)) @@ -64,14 +50,13 @@ export function interfaceDatastoreTests (test) { }) describe('putMany', () => { - /** @type {Datastore} */ - let store + let store: D beforeEach(async () => { store = await createStore() }) - afterEach(() => cleanup(store)) + afterEach(async () => { await cleanup(store) }) it('streaming', async () => { const data = [] @@ -94,14 +79,13 @@ export function interfaceDatastoreTests (test) { }) describe('get', () => { - /** @type {Datastore} */ - let store + let store: D beforeEach(async () => { store = await createStore() }) - afterEach(() => cleanup(store)) + afterEach(async () => { await cleanup(store) }) it('simple', async () => { const k = new Key('/z/one') @@ -125,14 +109,13 @@ export function interfaceDatastoreTests (test) { }) describe('getMany', () => { - /** @type {Datastore} */ - let store + let store: D beforeEach(async () => { store = await createStore() }) - afterEach(() => cleanup(store)) + afterEach(async () => { await cleanup(store) }) it('streaming', async () => { const k = new Key('/z/one') @@ -159,14 +142,13 @@ export function interfaceDatastoreTests (test) { }) describe('delete', () => { - /** @type {Datastore} */ - let store + let store: D beforeEach(async () => { store = await createStore() }) - afterEach(() => cleanup(store)) + afterEach(async () => { await cleanup(store) }) it('simple', async () => { const k = new Key('/z/one') @@ -178,33 +160,31 @@ export function interfaceDatastoreTests (test) { }) it('parallel', async () => { - /** @type {[Key, Uint8Array][]} */ - const data = [] + const data: Array<[Key, Uint8Array]> = [] for (let i = 0; i < 100; i++) { data.push([new Key(`/a/key${i}`), uint8ArrayFromString(`data${i}`)]) } - await Promise.all(data.map(d => store.put(d[0], d[1]))) + await Promise.all(data.map(async d => { await store.put(d[0], d[1]) })) - const res0 = await Promise.all(data.map(d => store.has(d[0]))) + const res0 = await Promise.all(data.map(async d => await store.has(d[0]))) res0.forEach(res => expect(res).to.be.eql(true)) - await Promise.all(data.map(d => store.delete(d[0]))) + await Promise.all(data.map(async d => { await store.delete(d[0]) })) - const res1 = await Promise.all(data.map(d => store.has(d[0]))) + const res1 = await Promise.all(data.map(async d => await store.has(d[0]))) res1.forEach(res => expect(res).to.be.eql(false)) }) }) describe('deleteMany', () => { - /** @type {Datastore} */ - let store + let store: D beforeEach(async () => { store = await createStore() }) - afterEach(() => cleanup(store)) + afterEach(async () => { await cleanup(store) }) it('streaming', async () => { const data = [] @@ -214,7 +194,7 @@ export function interfaceDatastoreTests (test) { await drain(store.putMany(data)) - const res0 = await Promise.all(data.map(d => store.has(d.key))) + const res0 = await Promise.all(data.map(async d => await store.has(d.key))) res0.forEach(res => expect(res).to.be.eql(true)) let index = 0 @@ -226,20 +206,19 @@ export function interfaceDatastoreTests (test) { expect(index).to.equal(data.length) - const res1 = await Promise.all(data.map(d => store.has(d.key))) + const res1 = await Promise.all(data.map(async d => await store.has(d.key))) res1.forEach(res => expect(res).to.be.eql(false)) }) }) describe('batch', () => { - /** @type {Datastore} */ - let store + let store: D beforeEach(async () => { store = await createStore() }) - afterEach(() => cleanup(store)) + afterEach(async () => { await cleanup(store) }) it('simple', async () => { const b = store.batch() @@ -253,7 +232,7 @@ export function interfaceDatastoreTests (test) { await b.commit() const keys = ['/a/one', '/q/two', '/q/three', '/z/old'] - const res = await Promise.all(keys.map(k => store.has(new Key(k)))) + const res = await Promise.all(keys.map(async k => await store.has(new Key(k)))) expect(res).to.be.eql([true, true, true, false]) }) @@ -270,53 +249,28 @@ export function interfaceDatastoreTests (test) { await b.commit() - /** - * @param {AsyncIterable} iterable - */ - const total = async iterable => { - let count = 0 - // eslint-disable-next-line no-unused-vars - for await (const _ of iterable) count++ - return count - } - - expect(await total(store.query({ prefix: '/a' }))).to.equal(count) - expect(await total(store.query({ prefix: '/z' }))).to.equal(count) - expect(await total(store.query({ prefix: '/q' }))).to.equal(count) + expect(await length(store.query({ prefix: '/a' }))).to.equal(count) + expect(await length(store.query({ prefix: '/z' }))).to.equal(count) + expect(await length(store.query({ prefix: '/q' }))).to.equal(count) }) }) describe('query', () => { - /** @type {Datastore} */ - let store + let store: D const hello = { key: new Key('/q/1hello'), value: uint8ArrayFromString('1') } const world = { key: new Key('/z/2world'), value: uint8ArrayFromString('2') } const hello2 = { key: new Key('/z/3hello2'), value: uint8ArrayFromString('3') } - /** - * @type {QueryFilter} - */ - const filter1 = entry => !entry.key.toString().endsWith('hello') - - /** - * @type {QueryFilter} - */ - const filter2 = entry => entry.key.toString().endsWith('hello2') + const filter1: QueryFilter = entry => !entry.key.toString().endsWith('hello') + const filter2: QueryFilter = entry => entry.key.toString().endsWith('hello2') - /** - * @type {QueryOrder} - */ - const order1 = (a, b) => { + const order1: QueryOrder = (a, b) => { if (a.value.toString() < b.value.toString()) { return -1 } return 1 } - - /** - * @type {QueryOrder} - */ - const order2 = (a, b) => { + const order2: QueryOrder = (a, b) => { if (a.value.toString() < b.value.toString()) { return 1 } @@ -326,8 +280,7 @@ export function interfaceDatastoreTests (test) { return 0 } - /** @type {Array<[string, any, any[]|number]>} */ - const tests = [ + const tests: Array<[string, any, any[] | number]> = [ ['empty', {}, [hello, world, hello2]], ['prefix', { prefix: '/z' }, [world, hello2]], ['1 filter', { filters: [filter1] }, [world, hello2]], @@ -347,10 +300,10 @@ export function interfaceDatastoreTests (test) { b.put(world.key, world.value) b.put(hello2.key, hello2.value) - return b.commit() + await b.commit() }) - after(() => cleanup(store)) + after(async () => { await cleanup(store) }) tests.forEach(([name, query, expected]) => it(name, async () => { let res = await all(store.query(query)) @@ -358,11 +311,8 @@ export function interfaceDatastoreTests (test) { if (Array.isArray(expected)) { if (query.orders == null) { expect(res).to.have.length(expected.length) - /** - * @param {Pair} a - * @param {Pair} b - */ - const s = (a, b) => { + + const s: QueryOrder = (a, b) => { if (a.key.toString() < b.key.toString()) { return 1 } else { @@ -423,36 +373,22 @@ export function interfaceDatastoreTests (test) { }) describe('queryKeys', () => { - /** @type {Datastore} */ - let store + let store: D const hello = { key: new Key('/q/1hello'), value: uint8ArrayFromString('1') } const world = { key: new Key('/z/2world'), value: uint8ArrayFromString('2') } const hello2 = { key: new Key('/z/3hello2'), value: uint8ArrayFromString('3') } - /** - * @type {KeyQueryFilter} - */ - const filter1 = key => !key.toString().endsWith('hello') + const filter1: KeyQueryFilter = key => !key.toString().endsWith('hello') + const filter2: KeyQueryFilter = key => key.toString().endsWith('hello2') - /** - * @type {KeyQueryFilter} - */ - const filter2 = key => key.toString().endsWith('hello2') - - /** - * @type {KeyQueryOrder} - */ - const order1 = (a, b) => { + const order1: KeyQueryOrder = (a, b) => { if (a.toString() < b.toString()) { return -1 } return 1 } - /** - * @type {KeyQueryOrder} - */ - const order2 = (a, b) => { + const order2: KeyQueryOrder = (a, b) => { if (a.toString() < b.toString()) { return 1 } @@ -462,8 +398,7 @@ export function interfaceDatastoreTests (test) { return 0 } - /** @type {Array<[string, any, any[]|number]>} */ - const tests = [ + const tests: Array<[string, any, any[] | number]> = [ ['empty', {}, [hello.key, world.key, hello2.key]], ['prefix', { prefix: '/z' }, [world.key, hello2.key]], ['1 filter', { filters: [filter1] }, [world.key, hello2.key]], @@ -483,10 +418,10 @@ export function interfaceDatastoreTests (test) { b.put(world.key, world.value) b.put(hello2.key, hello2.value) - return b.commit() + await b.commit() }) - after(() => cleanup(store)) + after(async () => { await cleanup(store) }) tests.forEach(([name, query, expected]) => it(name, async () => { let res = await all(store.queryKeys(query)) @@ -494,10 +429,8 @@ export function interfaceDatastoreTests (test) { if (Array.isArray(expected)) { if (query.orders == null) { expect(res).to.have.length(expected.length) - /** - * @type {KeyQueryOrder} - */ - const s = (a, b) => { + + const s: KeyQueryOrder = (a, b) => { if (a.toString() < b.toString()) { return 1 } else { @@ -550,23 +483,4 @@ export function interfaceDatastoreTests (test) { await writePromise }) }) - - describe('lifecycle', () => { - /** @type {Datastore} */ - let store - - before(async () => { - store = await test.setup() - if (!store) throw new Error('missing store') - }) - - after(() => cleanup(store)) - - it('close and open', async () => { - await store.close() - await store.open() - await store.close() - await store.open() - }) - }) } diff --git a/packages/interface-datastore-tests/tsconfig.json b/packages/interface-datastore-tests/tsconfig.json index edf3944f..ed6076a4 100644 --- a/packages/interface-datastore-tests/tsconfig.json +++ b/packages/interface-datastore-tests/tsconfig.json @@ -1,8 +1,7 @@ { "extends": "aegir/src/config/tsconfig.aegir.json", "compilerOptions": { - "outDir": "dist", - "emitDeclarationOnly": true + "outDir": "dist" }, "include": [ "src" diff --git a/packages/interface-datastore/src/index.ts b/packages/interface-datastore/src/index.ts index 88f7d920..042dd338 100644 --- a/packages/interface-datastore/src/index.ts +++ b/packages/interface-datastore/src/index.ts @@ -1,13 +1,6 @@ import type { - Pair as StorePair, - Batch as StoreBatch, - QueryFilter as StoreQueryFilter, - QueryOrder as StoreQueryOrder, - Query as StoreQuery, - KeyQueryFilter as StoreKeyQueryFilter, - KeyQueryOrder as StoreKeyQueryOrder, - KeyQuery as StoreKeyQuery, Options as StoreOptions, + Await, Store } from 'interface-store' import { Key } from './key.js' @@ -16,40 +9,86 @@ export interface Options extends StoreOptions { } -export interface Pair extends StorePair { - -} - -export interface Batch extends StoreBatch { - +export interface Pair { + key: Key + value: Uint8Array } -export interface Datastore extends Store { - -} - -export interface QueryFilter extends StoreQueryFilter { - +export interface Batch { + put: (key: Key, value: Uint8Array) => void + delete: (key: Key) => void + commit: (options?: Options) => Await } -export interface QueryOrder extends StoreQueryOrder { - -} - -export interface Query extends StoreQuery { - -} - -export interface KeyQueryFilter extends StoreKeyQueryFilter { - +export interface Datastore extends Store { + /** + * This will return an object with which you can chain multiple operations together, with them only being executed on calling `commit`. + * + * @example + * ```js + * const b = store.batch() + * + * for (let i = 0; i < 100; i++) { + * b.put(new Key(`hello${i}`), new TextEncoder('utf8').encode(`hello world ${i}`)) + * } + * + * await b.commit() + * console.log('put 100 values') + * ``` + */ + batch: () => Batch + + /** + * Query the datastore. + * + * @example + * ```js + * // retrieve __all__ key/value pairs from the store + * let list = [] + * for await (const { key, value } of store.query({})) { + * list.push(value) + * } + * console.log('ALL THE VALUES', list) + * ``` + */ + query: (query: Query, options?: Options) => AsyncIterable + + /** + * Query the datastore. + * + * @example + * ```js + * // retrieve __all__ keys from the store + * let list = [] + * for await (const key of store.queryKeys({})) { + * list.push(key) + * } + * console.log('ALL THE KEYS', key) + * ``` + */ + queryKeys: (query: KeyQuery, options?: Options) => AsyncIterable } -export interface KeyQueryOrder extends StoreKeyQueryOrder { +export interface QueryFilter { (item: Pair): boolean } +export interface QueryOrder { (a: Pair, b: Pair): -1 | 0 | 1 } +export interface Query { + prefix?: string + filters?: QueryFilter[] + orders?: QueryOrder[] + limit?: number + offset?: number } -export interface KeyQuery extends StoreKeyQuery { +export interface KeyQueryFilter { (item: Key): boolean } +export interface KeyQueryOrder { (a: Key, b: Key): -1 | 0 | 1 } +export interface KeyQuery { + prefix?: string + filters?: KeyQueryFilter[] + orders?: KeyQueryOrder[] + limit?: number + offset?: number } export { Key } diff --git a/packages/interface-store/src/index.ts b/packages/interface-store/src/index.ts index 4d60b676..5a4597ba 100644 --- a/packages/interface-store/src/index.ts +++ b/packages/interface-store/src/index.ts @@ -2,11 +2,6 @@ export type AwaitIterable = Iterable | AsyncIterable export type Await = Promise | T -export interface Pair { - key: Key - value: Value -} - /** * Options for async operations. */ @@ -14,39 +9,7 @@ export interface Options { signal?: AbortSignal } -export interface Batch { - put: (key: Key, value: Value) => void - delete: (key: Key) => void - commit: (options?: Options) => Promise -} - -export interface Store { - open: () => Promise - close: () => Promise - - /** - * Store the passed value under the passed key - * - * @example - * - * ```js - * await store.put([{ key: new Key('awesome'), value: new Uint8Array([0, 1, 2, 3]) }]) - * ``` - */ - put: (key: Key, val: Value, options?: Options) => Promise - - /** - * Retrieve the value stored under the given key - * - * @example - * ```js - * const value = await store.get(new Key('awesome')) - * console.log('got content: %s', value.toString('utf8')) - * // => got content: datastore - * ``` - */ - get: (key: Key, options?: Options) => Promise - +export interface Store { /** * Check for the existence of a value for the passed key * @@ -61,19 +24,18 @@ export interface Store { *} *``` */ - has: (key: Key, options?: Options) => Promise + has: (key: Key, options?: Options) => Await /** - * Remove the record for the passed key + * Store the passed value under the passed key * * @example * * ```js - * await store.delete(new Key('awesome')) - * console.log('deleted awesome content :(') + * await store.put([{ key: new Key('awesome'), value: new Uint8Array([0, 1, 2, 3]) }]) * ``` */ - delete: (key: Key, options?: Options) => Promise + put: (key: Key, val: Value, options?: Options) => Await /** * Store the given key/value pairs @@ -88,110 +50,65 @@ export interface Store { * ``` */ putMany: ( - source: AwaitIterable>, + source: AwaitIterable, options?: Options - ) => AsyncIterable> + ) => AwaitIterable /** - * Retrieve values for the passed keys + * Retrieve the value stored under the given key * * @example * ```js - * for await (const value of store.getMany([new Key('awesome')])) { - * console.log('got content:', new TextDecoder('utf8').decode(value)) - * // => got content: datastore - * } + * const value = await store.get(new Key('awesome')) + * console.log('got content: %s', value.toString('utf8')) + * // => got content: datastore * ``` */ - getMany: ( - source: AwaitIterable, - options?: Options - ) => AsyncIterable + get: (key: Key, options?: Options) => Await /** - * Remove values for the passed keys + * Retrieve values for the passed keys * * @example - * * ```js - * const source = [new Key('awesome')] - * - * for await (const key of store.deleteMany(source)) { - * console.log(`deleted content with key ${key}`) + * for await (const value of store.getMany([new Key('awesome')])) { + * console.log('got content:', new TextDecoder('utf8').decode(value)) + * // => got content: datastore * } * ``` */ - deleteMany: ( + getMany: ( source: AwaitIterable, options?: Options - ) => AsyncIterable + ) => AwaitIterable /** - * This will return an object with which you can chain multiple operations together, with them only being executed on calling `commit`. + * Remove the record for the passed key * * @example - * ```js - * const b = store.batch() - * - * for (let i = 0; i < 100; i++) { - * b.put(new Key(`hello${i}`), new TextEncoder('utf8').encode(`hello world ${i}`)) - * } * - * await b.commit() - * console.log('put 100 values') - * ``` - */ - batch: () => Batch - - /** - * Query the store. - * - * @example * ```js - * // retrieve __all__ key/value pairs from the store - * let list = [] - * for await (const { key, value } of store.query({})) { - * list.push(value) - * } - * console.log('ALL THE VALUES', list) + * await store.delete(new Key('awesome')) + * console.log('deleted awesome content :(') * ``` */ - query: (query: Query, options?: Options) => AsyncIterable> + delete: (key: Key, options?: Options) => Await /** - * Query the store. + * Remove values for the passed keys * * @example + * * ```js - * // retrieve __all__ keys from the store - * let list = [] - * for await (const key of store.queryKeys({})) { - * list.push(key) + * const source = [new Key('awesome')] + * + * for await (const key of store.deleteMany(source)) { + * console.log(`deleted content with key ${key}`) * } - * console.log('ALL THE KEYS', key) * ``` */ - queryKeys: (query: KeyQuery, options?: Options) => AsyncIterable -} - -export interface QueryFilter { (item: Pair): boolean } -export interface QueryOrder { (a: Pair, b: Pair): -1 | 0 | 1 } - -export interface Query { - prefix?: string - filters?: Array> - orders?: Array> - limit?: number - offset?: number -} - -export interface KeyQueryFilter { (item: Key): boolean } -export interface KeyQueryOrder { (a: Key, b: Key): -1 | 0 | 1 } - -export interface KeyQuery { - prefix?: string - filters?: Array> - orders?: Array> - limit?: number - offset?: number + deleteMany: ( + source: AwaitIterable, + options?: Options + ) => AwaitIterable }