diff --git a/packages/ipfs-core-types/src/pin/index.d.ts b/packages/ipfs-core-types/src/pin/index.d.ts index 9a22c8f6a7..37967244d3 100644 --- a/packages/ipfs-core-types/src/pin/index.d.ts +++ b/packages/ipfs-core-types/src/pin/index.d.ts @@ -1,6 +1,6 @@ import type { AbortOptions, AwaitIterable } from '../utils' import type CID from 'cids' -import type { API as Remote } from './remote' +import type { API as RemoteAPI } from './remote' export interface API { /** @@ -32,7 +32,7 @@ export interface API { * // CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u') * ``` */ - addAll: (source: AwaitIterable, options?: AddAllOptions & OptionExtension) => AsyncIterable + addAll: (source: PinSource, options?: AddAllOptions & OptionExtension) => AsyncIterable /** * List all the objects pinned to local storage @@ -90,9 +90,9 @@ export interface API { * // CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u') * ``` */ - rmAll: (source: AwaitIterable, options?: AbortOptions & OptionExtension) => AsyncIterable + rmAll: (source: PinSource, options?: AbortOptions & OptionExtension) => AsyncIterable - remote: Remote + remote: RemoteAPI } export interface AddOptions extends AbortOptions { @@ -124,50 +124,62 @@ export interface AddAllOptions extends AbortOptions { lock?: boolean } -export interface AddInput { - /** - * A CID to pin - nb. you must pass either `cid` or `path`, not both - */ - cid?: CID +export type Metadata = Record +export type ToPin = + | CID + | string + | ToPinWithCID + | ToPinWithPath +export interface ToPinWithPath extends PinDetails { /** - * An IPFS path to pin - nb. you must pass either `cid` or `path`, not both - */ - path?: string + * An IPFS path to pin. + */ + path: string + + cid?: undefined +} + +export interface ToPinWithCID extends PinDetails { + path?: undefined + cid: CID +} +export interface PinDetails { /** * If true, pin all blocked linked to from the pinned CID */ recursive?: boolean + metadata?: Metadata + /** * A human readable string to store with this pin */ comments?: string } -export type PinType = 'recursive' | 'direct' | 'indirect' | 'all' - -export type PinQueryType = 'recursive' | 'direct' | 'indirect' | 'all' - -export interface LsOptions extends AbortOptions { - paths?: CID | CID[] | string | string[] +export type PinSource = AwaitIterable + +export type PinType = + | 'recursive' + | 'direct' + | 'indirect' +export type PinQueryType = + | PinType + | 'all' +export interface LsOptions extends AbortOptions { + paths?: CID | string | Array type?: PinQueryType } export interface LsResult { cid: CID type: PinType | string - metadata?: Record + metadata?: Metadata } export interface RmOptions extends AbortOptions { recursive?: boolean } -export interface RmAllInput { - cid?: CID - path?: string - recursive?: boolean -} - diff --git a/packages/ipfs-core-types/src/utils.d.ts b/packages/ipfs-core-types/src/utils.d.ts index f8d59ac403..e63af0ef31 100644 --- a/packages/ipfs-core-types/src/utils.d.ts +++ b/packages/ipfs-core-types/src/utils.d.ts @@ -133,3 +133,57 @@ export interface BufferStore { get: (key: Uint8Array) => Promise stores: any[] } + +export interface Blockstore { + open: () => Promise + + /** + * Query the store + */ + query: (Query, options?: DatastoreOptions) => AsyncIterable + + /** + * Query the store, returning only keys + */ + queryKeys: (query: KeyQuery, options?: DatastoreOptions) => AsyncIterable + + /** + * Get a single block by CID + */ + get: (cid: CID, options?: DatastoreOptions) => Promise + + /** + * Like get, but for more + */ + getMany: (cids: AwaitIterable, options?: DatastoreOptions) => AsyncIterable + + /** + * Write a single block to the store + */ + put: (block: Block, options?: DatastoreOptions) => Promise + + /** + * Like put, but for more + */ + putMany: (blocks: AwaitIterable, options?: DatastoreOptions) => AsyncIterable + + /** + * Does the store contain block with this CID? + */ + has: (cid: CID, options?: DatastoreOptions) => Promise + + /** + * Delete a block from the store + */ + delete: (cid: CID, options?: DatastoreOptions) => Promise + + /** + * Delete a block from the store + */ + deleteMany: (cids: AwaitIterable, options?: DatastoreOptions) => AsyncIterable + + /** + * Close the store + */ + close: () => Promise +} diff --git a/packages/ipfs-core-utils/src/iterable.js b/packages/ipfs-core-utils/src/iterable.js new file mode 100644 index 0000000000..c98af3d9bd --- /dev/null +++ b/packages/ipfs-core-utils/src/iterable.js @@ -0,0 +1,23 @@ +'use strict' + +/** + * @template {Iterable} T + * @template U + * @param {T|U} value + * @returns {value is T} + */ +const isIterable = (value) => + // @ts-ignore + value && value[Symbol.iterator] + +/** + * @template {AsyncIterable} T + * @template U + * @param {T|U} value + * @returns {value is T} + */ +const isAsyncIterable = value => + // @ts-ignore + value && value[Symbol.asyncIterator] + +module.exports = { isIterable, isAsyncIterable } diff --git a/packages/ipfs-core-utils/src/pins/normalise-input.js b/packages/ipfs-core-utils/src/pins/normalise-input.js index e7dc21dd24..fb9682febf 100644 --- a/packages/ipfs-core-utils/src/pins/normalise-input.js +++ b/packages/ipfs-core-utils/src/pins/normalise-input.js @@ -2,6 +2,7 @@ const errCode = require('err-code') const CID = require('cids') +const { isIterable, isAsyncIterable } = require('../iterable') /** * @typedef {Object} Pinnable @@ -42,24 +43,33 @@ const CID = require('cids') * AsyncIterable<{ path: CID|String, recursive:boolean, metadata }> * ``` * - * @param {Source} input - * @returns {AsyncIterable} + * @param {import('ipfs-core-types/src/pin').ToPin|import('ipfs-core-types/src/pin').PinSource} input + * @returns {AsyncIterable} */ -// eslint-disable-next-line complexity -module.exports = async function * normaliseInput (input) { +async function * normaliseInput (input) { // must give us something if (input === null || input === undefined) { throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') } // CID|String - if (CID.isCID(input)) { - yield toPin({ cid: input }) + if (CID.isCID(input) || typeof input === 'string' || input instanceof String) { + return yield toPin(input) + } + + // Iterable + if (isIterable(input)) { + for (const each of input) { + yield toPin(each) + } return } - if (input instanceof String || typeof input === 'string') { - yield toPin({ path: input }) + // AsyncIterable + if (isAsyncIterable(input)) { + for await (const each of input) { + yield toPin(each) + } return } @@ -70,84 +80,30 @@ module.exports = async function * normaliseInput (input) { return yield toPin(input) } - // Iterable - if (Symbol.iterator in input) { - // @ts-ignore - const iterator = input[Symbol.iterator]() - const first = iterator.next() - if (first.done) return iterator - - // Iterable - if (CID.isCID(first.value) || first.value instanceof String || typeof first.value === 'string') { - yield toPin({ cid: first.value }) - for (const cid of iterator) { - yield toPin({ cid }) - } - return - } - - // Iterable<{ cid: CID recursive, metadata }> - if (first.value.cid != null || first.value.path != null) { - yield toPin(first.value) - for (const obj of iterator) { - yield toPin(obj) - } - return - } - - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') - } - - // AsyncIterable - if (Symbol.asyncIterator in input) { - // @ts-ignore - const iterator = input[Symbol.asyncIterator]() - const first = await iterator.next() - if (first.done) return iterator - - // AsyncIterable - if (CID.isCID(first.value) || first.value instanceof String || typeof first.value === 'string') { - yield toPin({ cid: first.value }) - for await (const cid of iterator) { - yield toPin({ cid }) - } - return - } - - // AsyncIterable<{ cid: CID|String recursive, metadata }> - if (first.value.cid != null || first.value.path != null) { - yield toPin(first.value) - for await (const obj of iterator) { - yield toPin(obj) - } - return - } - - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') - } - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') } /** - * @param {Pinnable} input + * @param {import('ipfs-core-types/src/pin').ToPin | InstanceType} input + * @returns {import('ipfs-core-types/src/pin').ToPinWithPath} */ -function toPin (input) { - const path = input.cid || `${input.path}` - - if (!path) { - throw errCode(new Error('Unexpected input: Please path either a CID or an IPFS path'), 'ERR_UNEXPECTED_INPUT') - } - - /** @type {Pin} */ - const pin = { - path, - recursive: input.recursive !== false - } - - if (input.metadata != null) { - pin.metadata = input.metadata +const toPin = (input) => { + if (typeof input === 'string') { + return { path: input, recursive: true } + } else if (input instanceof String) { + return { path: input.toString(), recursive: true } + } else if (CID.isCID(input)) { + return { path: input.toString(), recursive: true } + } else { + return { + path: `${input.path == null ? input.cid : input.path}`, + recursive: input.recursive !== false, + ...(input.metadata && { metadata: input.metadata }) + } } +} - return pin +module.exports = { + normaliseInput, + toPin } diff --git a/packages/ipfs-core-utils/test/pins/normalise-input.spec.js b/packages/ipfs-core-utils/test/pins/normalise-input.spec.js index 04e05d605d..23a6b50721 100644 --- a/packages/ipfs-core-utils/test/pins/normalise-input.spec.js +++ b/packages/ipfs-core-utils/test/pins/normalise-input.spec.js @@ -3,7 +3,7 @@ /* eslint-env mocha */ const { expect } = require('aegir/utils/chai') -const normalise = require('../../src/pins/normalise-input') +const { normaliseInput: normalise } = require('../../src/pins/normalise-input') const all = require('it-all') const CID = require('cids') diff --git a/packages/ipfs-core/src/components/pin/add-all.js b/packages/ipfs-core/src/components/pin/add-all.js index 6b5aa0ac58..72df50fe8a 100644 --- a/packages/ipfs-core/src/components/pin/add-all.js +++ b/packages/ipfs-core/src/components/pin/add-all.js @@ -5,73 +5,53 @@ const { resolvePath } = require('../../utils') const PinManager = require('./pin-manager') const { PinTypes } = PinManager const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') -const normaliseInput = require('ipfs-core-utils/src/pins/normalise-input') +const { normaliseInput } = require('ipfs-core-utils/src/pins/normalise-input') /** - * @typedef {import('ipfs-core-utils/src/pins/normalise-input').Source} Source - * @typedef {import('ipfs-core-utils/src/pins/normalise-input').Pin} PinTarget - * @typedef {import('ipfs-core-types/src/utils').AbortOptions} AbortOptions - * @typedef {import('cids')} CID - */ - -/** - * @template T - * @typedef {Iterable|AsyncIterable} AwaitIterable + * @param {import('.').Context} context + * @param {import('ipfs-core-types/src/pin').PinSource} source + * @param {import('ipfs-core-types/src/pin').AddOptions} [options] + * @returns {AsyncIterable} */ +async function * addAll (context, source, options = {}) { + // When adding a file, we take a lock that gets released after pinning + // is complete, so don't take a second lock here + if (options.lock) { + const release = await context.gcLock.readLock() + try { + yield * pinAdd(context, source) + } finally { + release() + } + } else { + yield * pinAdd(context, source) + } +} /** - * @param {Object} config - * @param {import('../gc-lock').GCLock} config.gcLock - * @param {import('ipld')} config.ipld - * @param {import('./pin-manager')} config.pinManager + * @param {import('.').Context} context + * @param {import('ipfs-core-types/src/pin').PinSource} source */ -module.exports = ({ pinManager, gcLock, ipld }) => { - /** - * @type {import('ipfs-core-types/src/pin').API["addAll"]} - */ - async function * addAll (source, options = {}) { - /** - * @returns {AsyncIterable} - */ - const pinAdd = async function * () { - for await (const { path, recursive, metadata } of normaliseInput(source)) { - const cid = await resolvePath(ipld, path) - - // verify that each hash can be pinned - const { reason } = await pinManager.isPinnedWithType(cid, [PinTypes.recursive, PinTypes.direct]) +async function * pinAdd ({ pinManager, ipld }, source) { + for await (const { path, recursive, metadata } of normaliseInput(source)) { + const cid = await resolvePath(ipld, path) - if (reason === 'recursive' && !recursive) { - // only disallow trying to override recursive pins - throw new Error(`${cid} already pinned recursively`) - } + // verify that each hash can be pinned + const { reason } = await pinManager.isPinnedWithType(cid, [PinTypes.recursive, PinTypes.direct]) - if (recursive) { - await pinManager.pinRecursively(cid, { metadata }) - } else { - await pinManager.pinDirectly(cid, { metadata }) - } - - yield cid - } + if (reason === 'recursive' && !recursive) { + // only disallow trying to override recursive pins + throw new Error(`${cid} already pinned recursively`) } - // When adding a file, we take a lock that gets released after pinning - // is complete, so don't take a second lock here - const lock = Boolean(options.lock) - - if (!lock) { - yield * pinAdd() - return + if (recursive) { + await pinManager.pinRecursively(cid, { metadata }) + } else { + await pinManager.pinDirectly(cid, { metadata }) } - const release = await gcLock.readLock() - - try { - yield * pinAdd() - } finally { - release() - } + yield cid } - - return withTimeoutOption(addAll) } + +module.exports = withTimeoutOption(addAll) diff --git a/packages/ipfs-core/src/components/pin/add.js b/packages/ipfs-core/src/components/pin/add.js index 842d34d166..e89ca74341 100644 --- a/packages/ipfs-core/src/components/pin/add.js +++ b/packages/ipfs-core/src/components/pin/add.js @@ -1,31 +1,20 @@ 'use strict' const last = require('it-last') -const CID = require('cids') - +const addAll = require('./add-all') +const { toPin } = require('ipfs-core-utils/src/pins/normalise-input') /** - * @param {Object} config - * @param {ReturnType} config.addAll + * @param {import('.').Context} context + * @param {import('cids')|string} path + * @param {import('ipfs-core-types/src/pin').AddOptions} [options] + * @returns {Promise} */ -module.exports = ({ addAll }) => - /** - * @type {import('ipfs-core-types/src/pin').API["add"]} - */ - (path, options = {}) => { - let iter - - if (CID.isCID(path)) { - iter = addAll([{ - cid: path, - ...options - }], options) - } else { - iter = addAll([{ - path: path.toString(), - ...options - }], options) - } +const add = async (context, path, options) => { + const source = [{ ...toPin(path), ...options }] + const cid = await last(addAll(context, source, options)) + // last of empty would be undefined, but here we know it won't be so we + // manually cast type. + return /** @type {import('cids')} */(cid) +} - // @ts-ignore return value of last can be undefined - return last(iter) - } +module.exports = add diff --git a/packages/ipfs-core/src/components/pin/index.js b/packages/ipfs-core/src/components/pin/index.js index b78d33f647..e550f3c7a3 100644 --- a/packages/ipfs-core/src/components/pin/index.js +++ b/packages/ipfs-core/src/components/pin/index.js @@ -1,45 +1,154 @@ 'use strict' -const createAdd = require('./add') -const createAddAll = require('./add-all') -const createLs = require('./ls') -const createRm = require('./rm') -const createRmAll = require('./rm-all') +const add = require('./add') +const addAll = require('./add-all') +const ls = require('./ls') +const rm = require('./rm') +const rmAll = require('./rm-all') /** - * @typedef {import('../gc-lock').GCLock} GCLock - * @typedef {import('./pin-manager')} PinManager + * @typedef {import('cids')} CID + * + * @typedef {Object} Context + * @property {import('../gc-lock').GCLock} gcLock + * @property {import('ipld')} ipld + * @property {import('./pin-manager')} pinManager */ +/** + * @typedef {import('ipfs-core-types/src/pin/index').API} API + * @implements {API} + */ class PinAPI { /** - * @param {Object} config - * @param {GCLock} config.gcLock - * @param {import('ipld')} config.ipld - * @param {PinManager} config.pinManager + * @param {Context} context */ constructor ({ gcLock, ipld, pinManager }) { - const addAll = createAddAll({ gcLock, ipld, pinManager }) - this.addAll = addAll - this.add = createAdd({ addAll }) - const rmAll = createRmAll({ gcLock, ipld, pinManager }) - this.rmAll = rmAll - this.rm = createRm({ rmAll }) - this.ls = createLs({ ipld, pinManager }) - - /** @type {import('ipfs-core-types/src/pin/remote').API} */ + this.gcLock = gcLock + this.ipld = ipld + this.pinManager = pinManager + this.remote = { - add: (cid, options = {}) => Promise.reject(new Error('Not implemented')), - ls: async function * (query, options = {}) { return Promise.reject(new Error('Not implemented')) }, // eslint-disable-line require-yield - rm: (query, options = {}) => Promise.reject(new Error('Not implemented')), - rmAll: (query, options = {}) => Promise.reject(new Error('Not implemented')), + add: notImplementedFn, + ls: notImplementedGn, + rm: notImplementedFn, + rmAll: notImplementedFn, service: { - add: (name, credentials) => Promise.reject(new Error('Not implemented')), - rm: (name, options = {}) => Promise.reject(new Error('Not implemented')), - ls: (options = {}) => Promise.reject(new Error('Not implemented')) + add: notImplementedFn, + rm: notImplementedFn, + ls: notImplementedFn } } } + + /** + * Adds an IPFS object to the pinset and also stores it to the IPFS repo. + * pinset is the set of hashes currently pinned (not gc'able) + * + * @param {CID|string} source + * @param {import('ipfs-core-types/src/pin').AddOptions} [options] + */ + + add (source, options) { + return add(this, source, options) + } + + /** + * Adds multiple IPFS objects to the pinset and also stores it to the IPFS + * repo. pinset is the set of hashes currently pinned (not gc'able) + * + * @example + * ```js + * const cid = CID.from('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u') + * for await (const cid of ipfs.pin.addAll([cid])) { + * console.log(cid) + * } + * // Logs: + * // CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u') + * ``` + * + * @param {import('ipfs-core-types/src/pin').PinSource} source - One or more CIDs or IPFS Paths to pin in your repo + * @param {import('ipfs-core-types/src/pin').AddAllOptions} [options] + */ + addAll (source, options) { + return addAll(this, source, options) + } + + /** + * Unpin this block from your repo + * + * @example + * ```js + * const cid = CID.from('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u') + * const result = await ipfs.pin.rm(cid) + * console.log(result) + * // prints the CID that was unpinned + * // CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u') + * ``` + * + * @param {CID|string} source + * @param {import('ipfs-core-types/src/pin').RmOptions} [options] + */ + rm (source, options) { + return rm(this, source, options) + } + + /** + * Unpin one or more blocks from your repo + * + * @example + * ```js + * const source = [ + * CID.from('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u') + * ] + * for await (const cid of ipfs.pin.rmAll(source)) { + * console.log(cid) + * } + * // prints the CIDs that were unpinned + * // CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u') + * ``` + * + * @param {import('ipfs-core-types/src/pin').PinSource} source + * @param {import('ipfs-core-types/src/pin').RmOptions} [options] + */ + rmAll (source, options) { + return rmAll(this, source, options) + } + + /** + * List all the objects pinned to local storage + * + * @example + * ```js + * for await (const { cid, type } of ipfs.pin.ls()) { + * console.log({ cid, type }) + * } + * // { cid: CID(Qmc5XkteJdb337s7VwFBAGtiaoj2QCEzyxtNRy3iMudc3E), type: 'recursive' } + * // { cid: CID(QmZbj5ruYneZb8FuR9wnLqJCpCXMQudhSdWhdhp5U1oPWJ), type: 'indirect' } + * // { cid: CID(QmSo73bmN47gBxMNqbdV6rZ4KJiqaArqJ1nu5TvFhqqj1R), type: 'indirect' } + * + * const paths = [ + * CID.from('Qmc5..'), + * CID.from('QmZb..'), + * CID.from('QmSo..') + * ] + * for await (const { cid, type } of ipfs.pin.ls({ paths })) { + * console.log({ cid, type }) + * } + * // { cid: CID(Qmc5XkteJdb337s7VwFBAGtiaoj2QCEzyxtNRy3iMudc3E), type: 'recursive' } + * // { cid: CID(QmZbj5ruYneZb8FuR9wnLqJCpCXMQudhSdWhdhp5U1oPWJ), type: 'indirect' } + * // { cid: CID(QmSo73bmN47gBxMNqbdV6rZ4KJiqaArqJ1nu5TvFhqqj1R), type: 'indirect' } + * ``` + * + * @param {import('ipfs-core-types/src/pin').LsOptions} [options] + */ + ls (options) { + return ls(this, options) + } } +const notImplementedFn = async () => { throw new Error('Not implemented') } +// eslint-disable-next-line require-yield +const notImplementedGn = async function * () { throw new Error('Not implemented') } + module.exports = PinAPI diff --git a/packages/ipfs-core/src/components/pin/ls.js b/packages/ipfs-core/src/components/pin/ls.js index b06b255fcc..c6910165ef 100644 --- a/packages/ipfs-core/src/components/pin/ls.js +++ b/packages/ipfs-core/src/components/pin/ls.js @@ -1,9 +1,8 @@ -/* eslint max-nested-callbacks: ["error", 8] */ 'use strict' const PinManager = require('./pin-manager') const { PinTypes } = PinManager -const normaliseInput = require('ipfs-core-utils/src/pins/normalise-input') +const { normaliseInput } = require('ipfs-core-utils/src/pins/normalise-input') const { resolvePath } = require('../../utils') const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') @@ -16,90 +15,77 @@ const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') * @param {CID} cid * @param {Record} [metadata] */ -function toPin (type, cid, metadata) { - /** @type {import('ipfs-core-types/src/pin').LsResult} */ - const output = { - type, - cid - } +const toPin = (type, cid, metadata) => ({ + type, + cid, + ...(metadata && { metadata }) +}) + +/** + * @param {import('.').Context} context + * @param {import('ipfs-core-types/src/pin').LsOptions} [options] + * @returns {AsyncIterable} + */ +async function * ls ({ pinManager, ipld }, options = {}) { + /** @type {import('ipfs-core-types/src/pin').PinQueryType} */ + let type = PinTypes.all + + if (options.type) { + type = options.type - if (metadata) { - output.metadata = metadata + PinManager.checkPinType(type) } - return output -} + if (options.paths) { + // check the pinned state of specific hashes + let matched = false -/** - * @param {Object} config - * @param {import('./pin-manager')} config.pinManager - * @param {import('ipld')} config.ipld - */ -module.exports = ({ pinManager, ipld }) => { - /** - * @type {import('ipfs-core-types/src/pin').API["ls"]} - */ - async function * ls (options = {}) { - /** @type {import('ipfs-core-types/src/pin').PinQueryType} */ - let type = PinTypes.all - - if (options.type) { - type = options.type - - PinManager.checkPinType(type) - } + for await (const { path } of normaliseInput(options.paths)) { + const cid = await resolvePath(ipld, path) + const { reason, pinned, parent, metadata } = await pinManager.isPinnedWithType(cid, type) - if (options.paths) { - // check the pinned state of specific hashes - let matched = false - - for await (const { path } of normaliseInput(options.paths)) { - const cid = await resolvePath(ipld, path) - const { reason, pinned, parent, metadata } = await pinManager.isPinnedWithType(cid, type) - - if (!pinned) { - throw new Error(`path '${path}' is not pinned`) - } - - switch (reason) { - case PinTypes.direct: - case PinTypes.recursive: - matched = true - yield toPin(reason, cid, metadata) - break - default: - matched = true - yield toPin(`${PinTypes.indirect} through ${parent}`, cid, metadata) - } + if (!pinned) { + throw new Error(`path '${path}' is not pinned`) } - if (!matched) { - throw new Error('No match found') + switch (reason) { + case PinTypes.direct: + case PinTypes.recursive: + matched = true + yield toPin(reason, cid, metadata) + break + default: + matched = true + yield toPin(`${PinTypes.indirect} through ${parent}`, cid, metadata) } - - return } - if (type === PinTypes.recursive || type === PinTypes.all) { - for await (const { cid, metadata } of pinManager.recursiveKeys()) { - yield toPin(PinTypes.recursive, cid, metadata) - } + if (!matched) { + throw new Error('No match found') } - if (type === PinTypes.indirect || type === PinTypes.all) { - // @ts-ignore - LsSettings & AbortOptions have no properties in common - // with type { preload?: boolean } - for await (const cid of pinManager.indirectKeys(options)) { - yield toPin(PinTypes.indirect, cid) - } + return + } + + if (type === PinTypes.recursive || type === PinTypes.all) { + for await (const { cid, metadata } of pinManager.recursiveKeys()) { + yield toPin(PinTypes.recursive, cid, metadata) } + } - if (type === PinTypes.direct || type === PinTypes.all) { - for await (const { cid, metadata } of pinManager.directKeys()) { - yield toPin(PinTypes.direct, cid, metadata) - } + if (type === PinTypes.indirect || type === PinTypes.all) { + // @ts-ignore - LsSettings & AbortOptions have no properties in common + // with type { preload?: boolean } + for await (const cid of pinManager.indirectKeys(options)) { + yield toPin(PinTypes.indirect, cid) } } - return withTimeoutOption(ls) + if (type === PinTypes.direct || type === PinTypes.all) { + for await (const { cid, metadata } of pinManager.directKeys()) { + yield toPin(PinTypes.direct, cid, metadata) + } + } } + +module.exports = withTimeoutOption(ls) diff --git a/packages/ipfs-core/src/components/pin/pin-manager.js b/packages/ipfs-core/src/components/pin/pin-manager.js index 0378cd8795..15f5744ec5 100644 --- a/packages/ipfs-core/src/components/pin/pin-manager.js +++ b/packages/ipfs-core/src/components/pin/pin-manager.js @@ -177,6 +177,7 @@ class PinManager { /** * @param {AbortOptions} [options] + * @returns {AsyncIterable<{ cid: CID, metadata: any }>} */ async * directKeys (options) { for await (const entry of this.repo.pins.query({ diff --git a/packages/ipfs-core/src/components/pin/rm-all.js b/packages/ipfs-core/src/components/pin/rm-all.js index 9e84fb4274..027524bf90 100644 --- a/packages/ipfs-core/src/components/pin/rm-all.js +++ b/packages/ipfs-core/src/components/pin/rm-all.js @@ -1,58 +1,53 @@ 'use strict' -const normaliseInput = require('ipfs-core-utils/src/pins/normalise-input') +const { normaliseInput } = require('ipfs-core-utils/src/pins/normalise-input') const { resolvePath } = require('../../utils') const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') const { PinTypes } = require('./pin-manager') /** - * @param {Object} config - * @param {import('./pin-manager')} config.pinManager - * @param {import('.').GCLock} config.gcLock - * @param {import('ipld')} config.ipld + * @param {import('.').Context} context + * @param {import('ipfs-core-types/src/pin').PinSource} source - Unpin all pins from the source + * @param {import('ipfs-core-types/src/pin').RmOptions} [_options] + * @returns {AsyncIterable} */ -module.exports = ({ pinManager, gcLock, ipld }) => { - /** - * @type {import('ipfs-core-types/src/pin').API["rmAll"]} - */ - async function * rmAll (source, _options = {}) { - const release = await gcLock.readLock() - - try { - // verify that each hash can be unpinned - for await (const { path, recursive } of normaliseInput(source)) { - const cid = await resolvePath(ipld, path) - const { pinned, reason } = await pinManager.isPinnedWithType(cid, PinTypes.all) - - if (!pinned) { - throw new Error(`${cid} is not pinned`) - } - - switch (reason) { - case (PinTypes.recursive): - if (!recursive) { - throw new Error(`${cid} is pinned recursively`) - } - - await pinManager.unpin(cid) - - yield cid - - break - case (PinTypes.direct): - await pinManager.unpin(cid) - - yield cid - - break - default: - throw new Error(`${cid} is pinned indirectly under ${reason}`) - } +async function * rmAll ({ pinManager, gcLock, ipld }, source, _options) { + const release = await gcLock.readLock() + + try { + // verify that each hash can be unpinned + for await (const { path, recursive } of normaliseInput(source)) { + const cid = await resolvePath(ipld, path) + const { pinned, reason } = await pinManager.isPinnedWithType(cid, PinTypes.all) + + if (!pinned) { + throw new Error(`${cid} is not pinned`) + } + + switch (reason) { + case (PinTypes.recursive): + if (!recursive) { + throw new Error(`${cid} is pinned recursively`) + } + + await pinManager.unpin(cid) + + yield cid + + break + case (PinTypes.direct): + await pinManager.unpin(cid) + + yield cid + + break + default: + throw new Error(`${cid} is pinned indirectly under ${reason}`) } - } finally { - release() } + } finally { + release() } - - return withTimeoutOption(rmAll) } + +module.exports = withTimeoutOption(rmAll) diff --git a/packages/ipfs-core/src/components/pin/rm.js b/packages/ipfs-core/src/components/pin/rm.js index 516cda8824..653e7e4cc9 100644 --- a/packages/ipfs-core/src/components/pin/rm.js +++ b/packages/ipfs-core/src/components/pin/rm.js @@ -1,16 +1,21 @@ 'use strict' const last = require('it-last') +const rmAll = require('./rm-all') +const { toPin } = require('ipfs-core-utils/src/pins/normalise-input') /** - * @param {Object} config - * @param {import('ipfs-core-types/src/pin').API["rmAll"]} config.rmAll + * @param {import('.').Context} context + * @param {string|import('cids')} path - CID or IPFS Path to unpin. + * @param {import('ipfs-core-types/src/pin').RmOptions} [options] + * @returns {Promise} - The CIDs that was unpinned */ -module.exports = ({ rmAll }) => - /** - * @type {import('ipfs-core-types/src/pin').API["rm"]} - */ - (path, options = {}) => { - // @ts-ignore return value of last can be undefined - return last(rmAll([{ path, ...options }], options)) - } +const rm = async (context, path, options = {}) => { + const source = [{ ...toPin(path), ...options }] + const cid = await last(rmAll(context, source, options)) + // last of empty would be undefined, but here we know it won't be so we + // manually cast type. + return /** @type {import('cids')} */(cid) +} + +module.exports = rm diff --git a/packages/ipfs-http-client/src/pin/add-all.js b/packages/ipfs-http-client/src/pin/add-all.js index fbfa2317b5..21d330a6ac 100644 --- a/packages/ipfs-http-client/src/pin/add-all.js +++ b/packages/ipfs-http-client/src/pin/add-all.js @@ -2,7 +2,7 @@ const CID = require('cids') const configure = require('../lib/configure') -const normaliseInput = require('ipfs-core-utils/src/pins/normalise-input') +const { normaliseInput } = require('ipfs-core-utils/src/pins/normalise-input') const toUrlSearchParams = require('../lib/to-url-search-params') /** diff --git a/packages/ipfs-http-client/src/pin/rm-all.js b/packages/ipfs-http-client/src/pin/rm-all.js index d4d0ca65b7..1055a1894b 100644 --- a/packages/ipfs-http-client/src/pin/rm-all.js +++ b/packages/ipfs-http-client/src/pin/rm-all.js @@ -2,7 +2,7 @@ const CID = require('cids') const configure = require('../lib/configure') -const normaliseInput = require('ipfs-core-utils/src/pins/normalise-input') +const { normaliseInput } = require('ipfs-core-utils/src/pins/normalise-input') const toUrlSearchParams = require('../lib/to-url-search-params') /** diff --git a/packages/ipfs-message-port-client/src/index.js b/packages/ipfs-message-port-client/src/index.js index fd01e814b8..437bed7661 100644 --- a/packages/ipfs-message-port-client/src/index.js +++ b/packages/ipfs-message-port-client/src/index.js @@ -6,6 +6,13 @@ const BlockClient = require('./block') const DAGClient = require('./dag') const CoreClient = require('./core') const FilesClient = require('./files') +const PinClient = require('./pin') + +/** + * @typedef {Object} ClientOptions + * @property {MessagePort} port + */ + class IPFSClient extends CoreClient { /** * @param {MessageTransport} transport @@ -13,9 +20,10 @@ class IPFSClient extends CoreClient { constructor (transport) { super(transport) this.transport = transport + this.block = new BlockClient(this.transport) this.dag = new DAGClient(this.transport) this.files = new FilesClient(this.transport) - this.block = new BlockClient(this.transport) + this.pin = new PinClient(this.transport) } /** diff --git a/packages/ipfs-message-port-client/src/pin.js b/packages/ipfs-message-port-client/src/pin.js new file mode 100644 index 0000000000..e8367e530b --- /dev/null +++ b/packages/ipfs-message-port-client/src/pin.js @@ -0,0 +1,142 @@ +'use strict' + +/* eslint-env browser */ +const CID = require('cids') +const Client = require('./client') +const { decodeCID } = require('ipfs-message-port-protocol/src/cid') +const { decodeIterable, encodeIterable } = require('ipfs-message-port-protocol/src/core') + +/** + * @typedef {import('./client').MessageTransport} MessageTransport + * @typedef {import('ipfs-message-port-server').PinService} PinService + * @typedef {import('ipfs-message-port-protocol/src/pin').API} API + */ + +/** + * @extends {Client} + * @implements {API} + */ +class PinClient extends Client { + /** + * @param {MessageTransport} transport + */ + constructor (transport) { + super('pin', ['add', 'ls', 'rm', 'rmAll'], transport) + } + + /** + * @param {string|CID} pathOrCID + * @param {import('ipfs-message-port-protocol/src/pin').AddOptions} [options] + * @returns {Promise} + */ + async add (pathOrCID, options = {}) { + const { cid } = await this.remote.add({ + ...options, + path: pathOrCID.toString() + }) + return decodeCID(cid) + } + + /** + * @param {import('ipfs-message-port-protocol/src/pin').ListOptions} [options] + */ + async * ls (options = {}) { + const paths = options.paths + // eslint-disable-next-line no-nested-ternary + const encodedPaths = paths === undefined + ? undefined + : Array.isArray(paths) + ? paths.map(String) + : [String(paths)] + + const result = await this.remote.ls({ + ...options, + paths: encodedPaths + }) + + yield * decodeIterable(result.data, decodePinEntry) + } + + /** + * @param {string|CID} source + * @param {import('ipfs-message-port-protocol/src/pin').RemoveOptions} options + */ + async rm (source, options = {}) { + const result = await this.remote.rm({ + ...options, + source: String(source) + }) + return decodeCID(result.cid) + } + + /** + * @param {import('ipfs-message-port-protocol/src/pin').PinSource} source + * @param {import('ipfs-message-port-protocol/src/pin').RemoveAllOptions} [options] + */ + async * rmAll (source, options = {}) { + const transfer = options.transfer || [] + const result = await this.remote.rmAll({ + ...options, + transfer, + source: encodeSource(source, transfer) + }) + + yield * decodeIterable(result.data, decodeCID) + } +} +module.exports = PinClient + +/** + * @param {import('ipfs-message-port-protocol/src/pin').PinSource} source + * @param {Transferable[]} transfer + * @returns {import('ipfs-message-port-protocol/src/pin').EncodedPinSource} + */ +const encodeSource = (source, transfer) => + encodeIterable(source, encodeToPin, transfer) + +/** + * @param {import('ipfs-core-types/src/pin').ToPin} value + * @returns {import('ipfs-message-port-protocol/src/pin').EncodedPin} + */ +const encodeToPin = (value) => { + if (CID.isCID(value)) { + return { + type: 'Pin', + path: value.toString() + } + } else if (typeof value === 'string') { + return { + type: 'Pin', + path: value + } + } else if (value instanceof String) { + return { + type: 'Pin', + path: value.toString() + } + } else if (value.cid) { + return { + ...value, + type: 'Pin', + cid: undefined, + path: value.cid.toString() + } + } else { + return { + ...value, + type: 'Pin', + cid: undefined, + path: value.path.toString() + } + } +} + +/** + * @param {import('ipfs-message-port-protocol/src/pin').EncodedPinEntry} entry + */ +const decodePinEntry = (entry) => { + return { + ...entry, + cid: decodeCID(entry.cid) + } +} diff --git a/packages/ipfs-message-port-client/test/interface-message-port-client.js b/packages/ipfs-message-port-client/test/interface-message-port-client.js index 9381c955dd..5c5efe68e2 100644 --- a/packages/ipfs-message-port-client/test/interface-message-port-client.js +++ b/packages/ipfs-message-port-client/test/interface-message-port-client.js @@ -148,11 +148,6 @@ describe('interface-ipfs-core tests', () => { name: 'should remove by CID in buffer', reason: 'Passing CID as Buffer is not supported' }, - { - name: 'should error when removing pinned blocks', - reason: 'ipfs.pin.add is not implemented' - }, - { name: 'should remove multiple CIDs', reason: 'times out' @@ -167,4 +162,42 @@ describe('interface-ipfs-core tests', () => { } ] }) + + tests.pin(factory, { + skip: [ + { + name: 'should add an array of CIDs', + reason: 'ipfs.pin.addAll is not implemented' + }, + { + name: 'should add a generator of CIDs', + reason: 'ipfs.pin.addAll is not implemented' + }, + { + name: 'should add an async generator of CIDs', + reason: 'ipfs.pin.addAll is not implemented' + }, + { + name: 'should add an array of pins with options', + reason: 'ipfs.pin.addAll is not implemented' + }, + { + name: 'should add a generator of pins with options', + reason: 'ipfs.pin.addAll is not implemented' + }, + { + name: 'should add an async generator of pins with options', + reason: 'ipfs.pin.addAll is not implemented' + }, + { + name: 'should respect timeout option when pinning a blocks', + reason: 'ipfs.pin.addAll is not implemented' + }, + // pin.rmAll + { + name: 'should pin dag-cbor with dag-pb child', + reason: 'ipfs.dag API does not handle DAGNode classes' + } + ] + }) }) diff --git a/packages/ipfs-message-port-protocol/src/pin.ts b/packages/ipfs-message-port-protocol/src/pin.ts new file mode 100644 index 0000000000..7f678a13c1 --- /dev/null +++ b/packages/ipfs-message-port-protocol/src/pin.ts @@ -0,0 +1,88 @@ +import type * as Pin from 'ipfs-core-types/src/pin' +import type { TransferOptions, QueryOptions } from './rpc' +import type { EncodedCID } from './cid' +import type CID from 'cids' +import type { RemoteIterable } from './core' + +export type LsResult = Pin.LsResult +export type PinSource = Pin.PinSource +export type PinType = Pin.PinType +export type PinQueryType = Pin.PinQueryType +export interface API { + add: (source: CID | string, options?: AddOptions) => Promise + // addAll: (source: PinSource, options?: AddAllOptions) => AsyncIterable + rm: (source: CID | string, options?: RemoveOptions) => Promise + rmAll: (source: PinSource, options?: RemoveAllOptions) => AsyncIterable + ls: (options?: ListOptions) => AsyncIterable +} + +export interface AddOptions extends Pin.AddOptions, TransferOptions { } +export interface AddAllOptions extends Pin.AddAllOptions, TransferOptions { } +export interface RemoveOptions extends Pin.RmOptions, TransferOptions { } +export interface RemoveAllOptions extends Pin.RmOptions, TransferOptions { } +export interface ListOptions extends Pin.LsOptions, TransferOptions { } + +export interface EncodedPin { + type: 'Pin' + path: string + cid?: undefined + recursive?: boolean + metadata?: any +} + +export type EncodedPinSource = RemoteIterable + +export interface EncodedPinEntry { + cid: EncodedCID + type: PinType + metadata?: any +} + +export interface AddQuery extends QueryOptions { + path: string + recursive?: boolean + matadata?: any +} + +export interface AddResult { + cid: EncodedCID + transfer: Transferable[] +} + +export interface ListQuery extends QueryOptions { + paths?: string[] + +} + +export interface ListResult { + data: RemoteIterable + transfer: Transferable[] +} + +export interface RemoveAllQuery extends QueryOptions { + source: EncodedPinSource + recursive?: boolean +} + +export interface RemoveResult { + cid: EncodedCID + transfer: Transferable[] +} + +export interface RemoveQuery extends QueryOptions { + source: string + recursive?: boolean +} + +export interface RemoveAllResult { + data: RemoteIterable + transfer: Transferable[] +} + +export interface Service { + add: (query: AddQuery) => Promise + ls: (query: ListQuery) => ListResult + + rm: (query: RemoveQuery) => Promise + rmAll: (query: RemoveAllQuery) => RemoveAllResult +} diff --git a/packages/ipfs-message-port-server/src/index.js b/packages/ipfs-message-port-server/src/index.js index 119aa05894..647f179fd3 100644 --- a/packages/ipfs-message-port-server/src/index.js +++ b/packages/ipfs-message-port-server/src/index.js @@ -16,5 +16,8 @@ exports.BlockService = BlockService const { IPFSService } = require('./service') exports.IPFSService = IPFSService +const { PinService } = require('./pin') +exports.PinService = PinService + const { Server } = require('./server') exports.Server = Server diff --git a/packages/ipfs-message-port-server/src/pin.js b/packages/ipfs-message-port-server/src/pin.js new file mode 100644 index 0000000000..00ff84d616 --- /dev/null +++ b/packages/ipfs-message-port-server/src/pin.js @@ -0,0 +1,113 @@ +'use strict' + +/* eslint-env browser */ + +const { encodeCID } = require('ipfs-message-port-protocol/src/cid') +const { decodeIterable, encodeIterable } = require('ipfs-message-port-protocol/src/core') + +/** + * @typedef {import('cids')} CID + * @typedef {import('ipfs-message-port-protocol/src/pin').Service} Service + * @typedef {import('ipfs-core-types').IPFS} IPFS + * + * @implements {Service} + */ + +exports.PinService = class PinService { + /** + * + * @param {IPFS} ipfs + */ + constructor (ipfs) { + this.ipfs = ipfs + } + + /** + * @param {import('ipfs-message-port-protocol/src/pin').AddQuery} query + */ + async add (query) { + const cid = await this.ipfs.pin.add(query.path, query) + /** @type {Transferable[]} */ + const transfer = [] + return { cid: encodeCID(cid, transfer), transfer } + } + + /** + * @param {import('ipfs-message-port-protocol/src/pin').ListQuery} query + */ + ls (query) { + const result = this.ipfs.pin.ls(query) + + return encodeListResult(result) + } + + /** + * @param {import('ipfs-message-port-protocol/src/pin').RemoveQuery} query + */ + async rm (query) { + const result = await this.ipfs.pin.rm(query.source, query) + /** @type {Transferable[]} */ + const transfer = [] + return { cid: encodeCID(result, transfer), transfer } + } + + /** + * @param {import('ipfs-message-port-protocol/src/pin').RemoveAllQuery} query + */ + rmAll (query) { + const { signal, source, timeout } = query + + const result = this.ipfs.pin.rmAll(decodeSource(source), { + signal, + timeout + }) + return encodeRmAllResult(result) + } +} + +/** + * @param {import('ipfs-message-port-protocol/src/pin').EncodedPinSource} source + */ +const decodeSource = (source) => decodeIterable(source, decodePin) + +/** + * + * @param {import('ipfs-message-port-protocol/src/pin').EncodedPin} pin + */ +const decodePin = pin => pin + +/** + * + * @param {AsyncIterable} entries + * @returns {import('ipfs-message-port-protocol/src/pin').ListResult} + */ +const encodeListResult = entries => { + /** @type {Transferable[]} */ + const transfer = [] + return { data: encodeIterable(entries, encodePinEntry, transfer), transfer } +} + +/** + * + * @param {AsyncIterable} entries + * @returns {import('ipfs-message-port-protocol/src/pin').RemoveAllResult} + */ +const encodeRmAllResult = entries => { + /** @type {Transferable[]} */ + const transfer = [] + return { data: encodeIterable(entries, encodeCID, transfer), transfer } +} + +/** + * @param {import('ipfs-core-types/src/pin').LsResult} entry + * @param {Transferable[]} _transfer + */ +const encodePinEntry = (entry, _transfer) => { + // Important: Looks like pin.ls sometimes yields cid + // which is referenced once again later, which is why + // we MUST NOT transfer CID or it gets corrupt. + return { + ...entry, + cid: encodeCID(entry.cid) + } +} diff --git a/packages/ipfs-message-port-server/src/service.js b/packages/ipfs-message-port-server/src/service.js index e56fb4038b..4eeb3dc6a3 100644 --- a/packages/ipfs-message-port-server/src/service.js +++ b/packages/ipfs-message-port-server/src/service.js @@ -2,10 +2,11 @@ /* eslint-env browser */ -const { DAGService } = require('./dag') +const { BlockService } = require('./block') const { CoreService } = require('./core') +const { DAGService } = require('./dag') const { FilesService } = require('./files') -const { BlockService } = require('./block') +const { PinService } = require('./pin') /** * @typedef {import('ipfs-core-types').IPFS} IPFS @@ -16,9 +17,10 @@ exports.IPFSService = class IPFSService { * @param {IPFS} ipfs */ constructor (ipfs) { - this.dag = new DAGService(ipfs) + this.block = new BlockService(ipfs) this.core = new CoreService(ipfs) + this.dag = new DAGService(ipfs) this.files = new FilesService(ipfs) - this.block = new BlockService(ipfs) + this.pin = new PinService(ipfs) } }