diff --git a/docs/core-api/PIN.md b/docs/core-api/PIN.md index 016f8d2c17..cd68eb3d52 100644 --- a/docs/core-api/PIN.md +++ b/docs/core-api/PIN.md @@ -25,6 +25,37 @@ - [Options](#options-4) - [Returns](#returns-4) - [Example](#example-4) +- [`ipfs.pin.remote.service.add(name, options)`](#ipfspinremoteserviceaddname-options) + - [Parameters](#parameters-5) + - [Options](#options-5) + - [Returns](#returns-5) + - [Example](#example-5) +- [`ipfs.pin.remote.service.ls([options])`](#ipfspinremoteservicels_options) + - [Options](#options-6) + - [Returns](#returns-6) + - [Example](#example-6) +- [`ipfs.pin.remote.service.rm(name, [options])`](#ipfspinremoteservicermname-options) + - [Parameters](#parameters-6) + - [Options](#options-7) + - [Returns](#returns-7) + - [Example](#example-7) +- [`ipfs.pin.remote.add(cid, [options])`](#ipfspinremoteaddcid-options) + - [Parameters](#parameters-7) + - [Options](#options-8) + - [Returns](#returns-8) + - [Example](#example-8) +- [`ipfs.pin.remote.ls(options)`](#ipfspinremotelsoptions) + - [Options](#options-9) + - [Returns](#returns-9) + - [Example](#example-9) +- [`ipfs.pin.remote.rm(options)`](#ipfspinremotermoptions) + - [Options](#options-10) + - [Returns](#returns-10) + - [Example](#example-10) +- [`ipfs.pin.remote.rmAll(options)`](#ipfspinremotermalloptions) + - [Options](#options-11) + - [Returns](#returns-11) + - [Example](#example-11) ## `ipfs.pin.add(ipfsPath, [options])` @@ -34,7 +65,7 @@ | Name | Type | Description | | ---- | ---- | ----------- | -| source | [CID][] or String | A CID or IPFS Path to pin in your repo | +| source | [CID][] or `string` | A CID or IPFS Path to pin in your repo | ### Options @@ -43,7 +74,7 @@ An optional object which may have the following keys: | Name | Type | Default | Description | | ---- | ---- | ------- | ----------- | | recursive | `boolean` | `true` | Recursively pin all links contained by the object | -| timeout | `Number` | `undefined` | A timeout in ms | +| timeout | `number` | `undefined` | A timeout in ms | | signal | [AbortSignal][] | `undefined` | Can be used to cancel any long running requests started as a result of this call | ### Returns @@ -71,7 +102,7 @@ A great source of [examples][] can be found in the tests for this API. | Name | Type | Description | | ---- | ---- | ----------- | -| source | `AsyncIterable<{ cid: CID, path: String, recursive: Boolean, comments: String }>` | One or more CIDs or IPFS Paths to pin in your repo | +| source | `AsyncIterable<{ cid: CID, path: string, recursive: boolean, comments: string }>` | One or more CIDs or IPFS Paths to pin in your repo | ### Options @@ -79,7 +110,7 @@ An optional object which may have the following keys: | Name | Type | Default | Description | | ---- | ---- | ------- | ----------- | -| timeout | `Number` | `undefined` | A timeout in ms | +| timeout | `number` | `undefined` | A timeout in ms | | signal | [AbortSignal][] | `undefined` | Can be used to cancel any long running requests started as a result of this call | ### Returns @@ -123,9 +154,9 @@ An optional object which may have the following keys: | Name | Type | Default | Description | | ---- | ---- | ------- | ----------- | -| paths | [CID][] or `Array` or `String` or `Array` | CIDs or IPFS paths to search for in the pinset | -| type | `String` | `undefined` | Filter by this type of pin ("recursive", "direct" or "indirect") | -| timeout | `Number` | `undefined` | A timeout in ms | +| paths | [CID][] or `Array` or `string` or `Array` | CIDs or IPFS paths to search for in the pinset | +| type | `string` | `undefined` | Filter by this type of pin ("recursive", "direct" or "indirect") | +| timeout | `number` | `undefined` | A timeout in ms | | signal | [AbortSignal][] | `undefined` | Can be used to cancel any long running requests started as a result of this call | ### Returns @@ -166,7 +197,7 @@ A great source of [examples][] can be found in the tests for this API. | Name | Type | Description | | ---- | ---- | ----------- | -| ipfsPath | [CID][] of String | Unpin this CID or IPFS Path | +| ipfsPath | [CID][] of string | Unpin this CID or IPFS Path | ### Options @@ -175,7 +206,7 @@ An optional object which may have the following keys: | Name | Type | Default | Description | | ---- | ---- | ------- | ----------- | | recursive | `boolean` | `true` | Recursively unpin the object linked | -| timeout | `Number` | `undefined` | A timeout in ms | +| timeout | `number` | `undefined` | A timeout in ms | | signal | [AbortSignal][] | `undefined` | Can be used to cancel any long running requests started as a result of this call | ### Returns @@ -203,7 +234,7 @@ A great source of [examples][] can be found in the tests for this API. | Name | Type | Description | | ---- | ---- | ----------- | -| source | [CID][], String or `AsyncIterable<{ cid: CID, path: String, recursive: Boolean }>` | Unpin this CID | +| source | [CID][], string or `AsyncIterable<{ cid: CID, path: string, recursive: boolean }>` | Unpin this CID | ### Options @@ -211,7 +242,7 @@ An optional object which may have the following keys: | Name | Type | Default | Description | | ---- | ---- | ------- | ----------- | -| timeout | `Number` | `undefined` | A timeout in ms | +| timeout | `number` | `undefined` | A timeout in ms | | signal | [AbortSignal][] | `undefined` | Can be used to cancel any long running requests started as a result of this call | ### Returns @@ -232,6 +263,377 @@ for await (const cid of ipfs.pin.rmAll(new CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH56 A great source of [examples][] can be found in the tests for this API. +## `ipfs.pin.remote.service.add(name, options)` + +> Registers remote pinning service with a given name. Errors if service with the given name is already registered. + +### Parameters + +| Name | Type | Description | +| ---- | ---- | ----------- | +| name | `string` | Service name | + +### Options + +An object which must contain following fields: + +| Name | Type | Description | +| ---- | ---- | ----------- | +| endpoint | `string` | Service endpoint URL | +| key | `string` | Service key | + + + +An object may have the following optional fields: + +| Name | Type | Default | Description | +| ---- | ---- | ------- | ----------- | +| timeout | `number` | `undefined` | A timeout in ms | +| signal | [AbortSignal][] | `undefined` | Can be used to cancel any long running requests started as a result of this call | + +### Returns + +| Type | Description | +| ---- | -------- | +| Promise | Resolves if added succesfully, or fails with error e.g. if service with such name is already registered | + + +### Example + +```JavaScript +await ipfs.pin.remote.sevice.add('pinata', { + endpoint: new URL('https://api.pinata.cloud'), + name: 'block-party' +}) +``` + +A great source of [examples][] can be found in the tests for this API. + + +## `ipfs.pin.remote.service.ls([options])` + +> List registered remote pinning services. + +### Options + +An object may have the following optional fields: + +| Name | Type | Default | Description | +| ---- | ---- | ------- | ----------- | +| stat | `boolean` | `false` | If `true` will include service stats. | +| timeout | `number` | `undefined` | A timeout in ms | +| signal | [AbortSignal][] | `undefined` | Can be used to cancel any long running requests started as a result of this call | + +### Returns + +| Type | Description | +| ---- | -------- | +| Promise<[RemotePinService][][]> | List of registered services | + +#### `RemotePinService` + +Object contains following fields: + +| Name | Type | Description | +| ---- | ---- | -------- | +| service | `string` | Service name | +| endpoint | `URL` | Service endpoint URL | +| stat | [Stat][] | Is included only when `stat: true` option was passed | + +#### `Stat` + +If stats could not be fetched from service (e.g. endpoint was unreachable) object has following form: + +| Name | Type | Description | +| ---- | ---- | -------- | +| status | `'invalid'` | Service status | + + +If stats were fetched from service succesfully object has following form: + +| Name | Type | Description | +| ---- | ---- | -------- | +| status | `'valid'` | Service status | +| pinCount | [PinCount][] | Pin counts | + +#### `PinCount` + +Object has following fields: + +| Name | Type | Description | +| ---- | ---- | ----------- | +| queued | `number` | Number of queued pins | +| pinning | `number` | Number of pins that are pinning | +| pinned | `number` | Number of pinned pins | +| failed | `number` | Number of faield pins | + + + +### Example + +```JavaScript +await ipfs.pin.remote.sevice.ls() +// [{ +// service: 'pinata' +// endpoint: new URL('https://api.pinata.cloud'), +// }] + +await ipfs.pin.remote.service.ls({ stat: true }) +// [{ +// service: 'pinata' +// endpoint: new URL('https://api.pinata.cloud'), +// stat: { +// status: 'valid', +// pinCount: { +// queued: 0, +// pinning: 0, +// pinned: 1, +// failed: 0, +// } +// } +// }] +``` + +A great source of [examples][] can be found in the tests for this API. + + +## `ipfs.pin.remote.service.rm(name, [options])` + +> Unregisteres remote pinning service with a given name (if service with such name is regisetered). + +### Parameters + +| Name | Type | Description | +| ---- | ---- | ----------- | +| name | `string` | Service name | + +### Options + +An object may have the following optional fields: + +| Name | Type | Default | Description | +| ---- | ---- | ------- | ----------- | +| timeout | `number` | `undefined` | A timeout in ms | +| signal | [AbortSignal][] | `undefined` | Can be used to cancel any long running requests started as a result of this call | + +### Returns + +| Type | Description | +| ---- | -------- | +| Promise | Resolves on completion | + + +### Example + +```JavaScript +await ipfs.pin.remote.sevice.rm('pinata') +``` + +A great source of [examples][] can be found in the tests for this API. + + +## `ipfs.pin.remote.add(cid, [options])` + +> Pin a content with a given CID to a remote pinning service + +### Parameters + +| Name | Type | Description | +| ---- | ---- | ----------- | +| cid | [CID][] | A CID to pin on a remote pinning service | + +### Options + +An object which must contain following fields: + +| Name | Type | Description | +| ---- | ---- | ----------- | +| service | `string` | Name of the remote pinning service to use | + + +An object may have the following optional fields: + +| Name | Type | Default | Description | +| ---- | ---- | ------- | ----------- | +| name | `string` | `undefined` | Name for pinned data; can be used for lookups later (max 255 characters) | +| origins | `Multiaddr[]` | `undefined` | List of multiaddrs known to provide the data (max 20) | +| background | `boolean` | `false` | If true, will add to the queue on the remote service and return immediately. If false or omitted will wait until pinned on the remote service | +| timeout | `number` | `undefined` | A timeout in ms | +| signal | [AbortSignal][] | `undefined` | Can be used to cancel any long running requests started as a result of this call | + +### Returns + +| Type | Description | +| ---- | -------- | +| [Pin][] | Pin Object | + +#### `Pin` + +Object has following fields: + +| Type | Description | +| ---- | ----------- | +| [Status][] | Pin status | +| [CID][] | CID of the content | +| `string | undefined` | name that was given to the pin, or `undefined` if no name was not given | + +#### `Status` + +Status is one of the following string values: + +`'queued'`, `'pinning'`, `'pinned'`, `'failed'` + +### Example + +```JavaScript +const cid = new CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u') +const pin = await ipfs.pin.remote.add(cid, { + service: 'pinata', + name: 'block-party' +}) +console.log(pin) +// Logs: +// { +// status: 'pinned', +// cid: CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u'), +// name: 'block-party' +// } +``` + +A great source of [examples][] can be found in the tests for this API. + +## `ipfs.pin.remote.ls(options)` + +> Returns a list of matching pins on the remote pinning service. + + +### Options + +An object which must contain following fields: + +| Name | Type | Description | +| ---- | ---- | ----------- | +| service | `string` | Name of the remote pinning service to use | + +An object may have the following optional fields: + +| Name | Type | Default | Description | +| ---- | ---- | ------- | ----------- | +| cid | [CID][][] | `undefined` | If provided, will only include pin objects that have a CID from the given set. | +| name | `string` | `undefined` | If passed, will only include pin objects with names that have this name (case-sensitive, exact match). | +| status | [Status][][] | ['pinned'] | Return pin objects for pins that have one of the specified status values | +| timeout | `number` | `undefined` | A timeout in ms | +| signal | [AbortSignal][] | `undefined` | Can be used to cancel any long running requests started as a result of this call | + +### Returns + +| Type | Description | +| ---- | -------- | +| AyncIterable<[Pin][]> | Pin Objects | + +### Example + +```JavaScript +const pins = await ipfs.pin.remote.ls({ + service: 'pinata' +}) +console.log(pins) +// Logs: +// [{ +// status: 'pinned', +// cid: CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u'), +// name: 'block-party' +// }] +``` + +A great source of [examples][] can be found in the tests for this API. + +## `ipfs.pin.remote.rm(options)` + +> Removes a single matching pin object from the remote pinning service. Will error when multiple pins mtach, to remove all matches `rmAll` should be used instead. + +### Options + +An object which must contain following fields: + +| Name | Type | Description | +| ---- | ---- | ----------- | +| service | `string` | Name of the remote pinning service to use | + +An object may also contain following optional fields: + +| Name | Type | Default | Description | +| ---- | ---- | ------- | ----------- | +| cid | [CID][][] | `undefined` | If provided, will match pin object(s) that have a CID from the given set. | +| name | `string` | `undefined` | If provided, will match pin object(s) with exact (case-sensitive) name. | +| status | [Status][][] | ['pinned'] | If provided, will match pin object(s) that have a status from the given set. | +| timeout | `number` | `undefined` | A timeout in ms | +| signal | [AbortSignal][] | `undefined` | Can be used to cancel any long running requests started as a result of this call | + +### Returns + +| Type | Description | +| ---- | -------- | +| Promise | Succeeds on completion | + +### Example + +```JavaScript +await ipfs.pin.remote.rm({ + service: 'pinata', + name: 'block-party' +}) +``` + +A great source of [examples][] can be found in the tests for this API. + +## `ipfs.pin.remote.rmAll(options)` + +> Removes all the matching pin objects from the remote pinning +service. + +### Options + +An object which must contain following fields: + +| Name | Type | Description | +| ---- | ---- | ----------- | +| service | `string` | Name of the remote pinning service to use | + +An object may also contain following optional fields: + +| Name | Type | Default | Description | +| ---- | ---- | ------- | ----------- | +| cid | [CID][][] | `undefined` | If provided, will match pin object(s) that have a CID from the given set. | +| name | `string` | `undefined` | If provided, will match pin object(s) with exact (case-sensitive) name. | +| status | [Status][][] | ['pinned'] | If provided, will match pin object(s) that have a status from the given set. | +| timeout | `number` | `undefined` | A timeout in ms | +| signal | [AbortSignal][] | `undefined` | Can be used to cancel any long running requests started as a result of this call | + +### Returns + +| Type | Description | +| ---- | -------- | +| Promise | Succeeds on completion | + +### Example + +```JavaScript +// Delete all non 'pinned' pins +await ipfs.pin.remote.rmAll({ + service: 'pinata', + status: ['queued', 'pinning', 'failed'] +}) +``` + +A great source of [examples][] can be found in the tests for this API. + +[Pin]: #pin +[Status]: #status +[RemotePinService]: #remotepinservice +[Status]: #status +[Stat]: #stat +[PinCount]: #pincount [examples]: https://github.com/ipfs/js-ipfs/blob/master/packages/interface-ipfs-core/src/pin [cid]: https://www.npmjs.com/package/cids [AbortSignal]: https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal diff --git a/examples/browser-ipns-publish/package.json b/examples/browser-ipns-publish/package.json index d62803afd1..5964876fca 100644 --- a/examples/browser-ipns-publish/package.json +++ b/examples/browser-ipns-publish/package.json @@ -28,7 +28,7 @@ "delay": "^4.4.0", "execa": "^4.0.3", "ipfsd-ctl": "^7.2.0", - "go-ipfs": "^0.7.0", + "go-ipfs": "0.8.0-rc2", "parcel-bundler": "^1.12.4", "path": "^0.12.7", "test-ipfs-example": "^2.0.3" diff --git a/examples/http-client-browser-pubsub/package.json b/examples/http-client-browser-pubsub/package.json index 749f4d0314..0c24d53a4b 100644 --- a/examples/http-client-browser-pubsub/package.json +++ b/examples/http-client-browser-pubsub/package.json @@ -19,7 +19,7 @@ ], "devDependencies": { "execa": "^4.0.3", - "go-ipfs": "^0.7.0", + "go-ipfs": "0.8.0-rc2", "ipfs": "^0.53.2", "ipfsd-ctl": "^7.2.0", "parcel-bundler": "^1.12.4", diff --git a/examples/http-client-name-api/package.json b/examples/http-client-name-api/package.json index 11bb7ff478..6503643c5e 100644 --- a/examples/http-client-name-api/package.json +++ b/examples/http-client-name-api/package.json @@ -17,7 +17,7 @@ }, "devDependencies": { "execa": "^4.0.3", - "go-ipfs": "^0.7.0", + "go-ipfs": "0.8.0-rc2", "ipfsd-ctl": "^7.2.0", "parcel-bundler": "^1.12.4", "rimraf": "^3.0.2", diff --git a/packages/interface-ipfs-core/src/index.js b/packages/interface-ipfs-core/src/index.js index 6ce8693ad5..239bcf60c7 100644 --- a/packages/interface-ipfs-core/src/index.js +++ b/packages/interface-ipfs-core/src/index.js @@ -20,6 +20,7 @@ exports.block = require('./block') exports.dag = require('./dag') exports.object = require('./object') exports.pin = require('./pin') +exports.pin.remote = require('./pin/remote') exports.bootstrap = require('./bootstrap') exports.dht = require('./dht') diff --git a/packages/interface-ipfs-core/src/pin/remote/add.js b/packages/interface-ipfs-core/src/pin/remote/add.js new file mode 100644 index 0000000000..78457490f9 --- /dev/null +++ b/packages/interface-ipfs-core/src/pin/remote/add.js @@ -0,0 +1,149 @@ +/* eslint-env mocha */ +'use strict' + +const { fixtures, clearRemotePins, clearServices } = require('../utils') +const { getDescribe, getIt, expect } = require('../../utils/mocha') +const testTimeout = require('../../utils/test-timeout') +const CID = require('cids') + +/** @typedef { import("ipfsd-ctl/src/factory") } Factory */ +/** + * @param {Factory} common + * @param {Object} options + */ +module.exports = (common, options) => { + const describe = getDescribe(options) + const it = getIt(options) + + const ENDPOINT = new URL(process.env.PINNING_SERVICE_ENDPOINT || '') + const KEY = process.env.PINNING_SERVIEC_KEY + const SERVICE = 'pinbot' + + describe('.pin.remote.add', function () { + this.timeout(50 * 1000) + + let ipfs + before(async () => { + ipfs = (await common.spawn()).api + await ipfs.pin.remote.service.add(SERVICE, { + endpoint: ENDPOINT, + key: KEY + }) + }) + after(async () => { + await clearServices(ipfs) + await common.clean() + }) + + beforeEach(async () => { + await clearRemotePins(ipfs) + }) + + it('should add a CID and return the added CID', async () => { + const pin = await ipfs.pin.remote.add(fixtures.files[0].cid, { + name: 'fixtures-files-0', + background: true, + service: SERVICE + }) + + expect(pin).to.deep.equal({ + status: 'queued', + cid: fixtures.files[0].cid, + name: 'fixtures-files-0' + }) + }) + + it('should fail if service is not provided', async () => { + const result = ipfs.pin.remote.add(fixtures.files[0].cid, { + name: 'fixtures-files-0', + background: true + }) + + await expect(result).to.eventually.be.rejectedWith(/service name must be passed/) + }) + + it('if name is not provided defaults to ""', async () => { + const pin = await ipfs.pin.remote.add(fixtures.files[0].cid, { + background: true, + service: SERVICE + }) + + expect(pin).to.deep.equal({ + cid: fixtures.files[0].cid, + name: '', + status: 'queued' + }) + }) + + it('should default to blocking pin', async () => { + const { cid } = fixtures.files[0] + const result = ipfs.pin.remote.add(cid, { + service: SERVICE + }) + + const timeout = {} + + const winner = await Promise.race([ + result, + new Promise(resolve => setTimeout(resolve, 100, timeout)) + ]) + + expect(winner).to.equal(timeout) + + // trigger status change on the mock service + ipfs.pin.remote.add(cid, { + service: SERVICE, + name: 'pinned-block' + }) + + expect(await result).to.deep.equal({ + cid, + status: 'pinned', + name: '' + }) + }) + it('should pin dag-cbor', async () => { + const cid = await ipfs.dag.put({}, { + format: 'dag-cbor', + hashAlg: 'sha2-256' + }) + + const pin = await ipfs.pin.remote.add(cid, { + service: SERVICE, + name: 'cbor-pin', + background: true + }) + + expect(pin).to.deep.equal({ + cid, + name: 'cbor-pin', + status: 'queued' + }) + }) + + it('should pin raw', async () => { + const cid = await ipfs.dag.put(new Uint8Array(0), { + format: 'raw', + hashAlg: 'sha2-256' + }) + + const pin = await ipfs.pin.remote.add(cid, { + service: SERVICE, + background: true + }) + + expect(pin).to.deep.equal({ + cid, + status: 'queued', + name: '' + }) + }) + + it('should respect timeout option when pinning a block', () => { + return testTimeout(() => ipfs.pin.remote.add(new CID('Qmd7qZS4T7xXtsNFdRoK1trfMs5zU94EpokQ9WFtxdPxsZ'), { + timeout: 1, + service: SERVICE + })) + }) + }) +} diff --git a/packages/interface-ipfs-core/src/pin/remote/index.js b/packages/interface-ipfs-core/src/pin/remote/index.js new file mode 100644 index 0000000000..81029db664 --- /dev/null +++ b/packages/interface-ipfs-core/src/pin/remote/index.js @@ -0,0 +1,12 @@ +'use strict' +const { createSuite } = require('../../utils/suite') + +const tests = { + service: require('./service'), + add: require('./add'), + ls: require('./ls'), + rm: require('./rm'), + rmAll: require('./rm-all') +} + +module.exports = createSuite(tests) diff --git a/packages/interface-ipfs-core/src/pin/remote/ls.js b/packages/interface-ipfs-core/src/pin/remote/ls.js new file mode 100644 index 0000000000..b3a92f4177 --- /dev/null +++ b/packages/interface-ipfs-core/src/pin/remote/ls.js @@ -0,0 +1,434 @@ +/* eslint-env mocha */ +'use strict' + +const { clearRemotePins, addRemotePins, clearServices } = require('../utils') +const { getDescribe, getIt, expect } = require('../../utils/mocha') +const all = require('it-all') +const testTimeout = require('../../utils/test-timeout') +const CID = require('cids') + +/** @typedef { import("ipfsd-ctl/src/factory") } Factory */ +/** + * @param {Factory} common + * @param {Object} options + */ +module.exports = (common, options) => { + const describe = getDescribe(options) + const it = getIt(options) + + const ENDPOINT = new URL(process.env.PINNING_SERVICE_ENDPOINT || '') + const KEY = process.env.PINNING_SERVIEC_KEY + const SERVICE = 'pinbot' + + const cid1 = new CID('QmbKtKBrmeRHjNCwR4zAfCJdMVu6dgmwk9M9AE9pUM9RgG') + const cid2 = new CID('QmdFyxZXsFiP4csgfM5uPu99AvFiKH62CSPDw5TP92nr7w') + const cid3 = new CID('Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP') + const cid4 = new CID('QmY9cxiHqTFoWamkQVkpmmqzBrY3hCBEL2XNu3NtX74Fuu') + + describe('.pin.remote.ls', function () { + this.timeout(50 * 1000) + + let ipfs + before(async () => { + ipfs = (await common.spawn()).api + await ipfs.pin.remote.service.add(SERVICE, { + endpoint: ENDPOINT, + key: KEY + }) + }) + after(async () => { + await clearServices(ipfs) + await common.clean() + }) + + beforeEach(async () => { + await clearRemotePins(ipfs) + }) + + it('requires service option', async () => { + const result = ipfs.pin.remote.ls({}) + await expect(all(result)).to.eventually.be.rejectedWith(/service name must be passed/) + }) + + it('list no pins', async () => { + const result = ipfs.pin.remote.ls({ service: SERVICE }) + const pins = await all(result) + expect(pins).to.deep.equal([]) + }) + + describe('list pins by status', () => { + it('list only pinned pins by default', async () => { + await addRemotePins(ipfs, SERVICE, { + one: cid1, + 'pinned-two': cid2, + 'pinning-three': cid3, + 'failed-four': cid4 + }) + + const list = await all(ipfs.pin.remote.ls({ + service: SERVICE + })) + + expect(list).to.deep.equal([ + { + status: 'pinned', + cid: cid2, + name: 'pinned-two' + } + ]) + }) + + it('should list "queued" pins', async () => { + await addRemotePins(ipfs, SERVICE, { + one: cid1, + 'pinned-two': cid2, + 'pinning-three': cid3, + 'failed-four': cid4 + }) + + const list = await all(ipfs.pin.remote.ls({ + status: ['queued'], + service: SERVICE + })) + + expect(list).to.deep.equal([ + { + status: 'queued', + cid: cid1, + name: 'one' + } + ]) + }) + + it('should list "pinning" pins', async () => { + await addRemotePins(ipfs, SERVICE, { + one: cid1, + 'pinned-two': cid2, + 'pinning-three': cid3, + 'failed-four': cid4 + }) + + const list = await all(ipfs.pin.remote.ls({ + status: ['pinning'], + service: SERVICE + })) + + expect(list).to.deep.equal([ + { + status: 'pinning', + cid: cid3, + name: 'pinning-three' + } + ]) + }) + + it('should list "failed" pins', async () => { + await addRemotePins(ipfs, SERVICE, { + one: cid1, + 'pinned-two': cid2, + 'pinning-three': cid3, + 'failed-four': cid4 + }) + + const list = await all(ipfs.pin.remote.ls({ + status: ['failed'], + service: SERVICE + })) + + expect(list).to.deep.equal([ + { + status: 'failed', + cid: cid4, + name: 'failed-four' + } + ]) + }) + + it('should list queued+pinned pins', async () => { + await addRemotePins(ipfs, SERVICE, { + one: cid1, + 'pinned-two': cid2, + 'pinning-three': cid3, + 'failed-four': cid4 + }) + + const list = await all(ipfs.pin.remote.ls({ + status: ['queued', 'pinned'], + service: SERVICE + })) + + expect(list.sort(byCID)).to.deep.equal([ + { + status: 'queued', + cid: cid1, + name: 'one' + }, + { + status: 'pinned', + cid: cid2, + name: 'pinned-two' + } + ].sort(byCID)) + }) + + it('should list queued+pinned+pinning pins', async () => { + await addRemotePins(ipfs, SERVICE, { + one: cid1, + 'pinned-two': cid2, + 'pinning-three': cid3, + 'failed-four': cid4 + }) + + const list = await all(ipfs.pin.remote.ls({ + status: ['queued', 'pinned', 'pinning'], + service: SERVICE + })) + + expect(list.sort(byCID)).to.deep.equal([ + { + status: 'queued', + cid: cid1, + name: 'one' + }, + { + status: 'pinned', + cid: cid2, + name: 'pinned-two' + }, + { + status: 'pinning', + cid: cid3, + name: 'pinning-three' + } + ].sort(byCID)) + }) + + it('should list queued+pinned+pinning+failed pins', async () => { + await addRemotePins(ipfs, SERVICE, { + one: cid1, + 'pinned-two': cid2, + 'pinning-three': cid3, + 'failed-four': cid4 + }) + + const list = await all(ipfs.pin.remote.ls({ + status: ['queued', 'pinned', 'pinning', 'failed'], + service: SERVICE + })) + + expect(list.sort(byCID)).to.deep.equal([ + { + status: 'queued', + cid: cid1, + name: 'one' + }, + { + status: 'pinned', + cid: cid2, + name: 'pinned-two' + }, + { + status: 'pinning', + cid: cid3, + name: 'pinning-three' + }, + { + status: 'failed', + cid: cid4, + name: 'failed-four' + } + ].sort(byCID)) + }) + }) + + describe('list pins by name', () => { + it('should list no pins when names do not match', async () => { + await addRemotePins(ipfs, SERVICE, { + a: cid1, + b: cid2, + c: cid3 + }) + + const list = await all(ipfs.pin.remote.ls({ + name: 'd', + status: ['queued', 'pinning', 'pinned', 'failed'], + service: SERVICE + })) + + expect(list).to.deep.equal([]) + }) + it('should list only pins with matchin names', async () => { + await addRemotePins(ipfs, SERVICE, { + a: cid1, + b: cid2 + }) + await addRemotePins(ipfs, SERVICE, { + a: cid3, + b: cid4 + }) + + const list = await all(ipfs.pin.remote.ls({ + name: 'a', + status: ['queued', 'pinning', 'pinned', 'failed'], + service: SERVICE + })) + + expect(list.sort(byCID)).to.deep.equal([ + { + status: 'queued', + name: 'a', + cid: cid1 + }, + { + status: 'queued', + name: 'a', + cid: cid3 + } + ].sort(byCID)) + }) + + it('should list only pins with matchin names & status', async () => { + await addRemotePins(ipfs, SERVICE, { + a: cid1, + b: cid2 + }) + await addRemotePins(ipfs, SERVICE, { + a: cid3, + b: cid4 + }) + // update status + await addRemotePins(ipfs, SERVICE, { + 'pinned-a': cid3 + }) + + const list = await all(ipfs.pin.remote.ls({ + name: 'a', + status: ['pinned'], + service: SERVICE + })) + + expect(list).to.deep.equal([ + { + status: 'pinned', + name: 'a', + cid: cid3 + } + ]) + }) + }) + + describe('list pins by cid', () => { + it('should list pins with matching cid', async () => { + await addRemotePins(ipfs, SERVICE, { + a: cid1, + b: cid2, + c: cid3, + d: cid4 + }) + + const list = await all(ipfs.pin.remote.ls({ + cid: [cid1], + status: ['queued', 'pinned', 'pinning', 'failed'], + service: SERVICE + })) + + expect(list).to.deep.equal([ + { + status: 'queued', + cid: cid1, + name: 'a' + } + ]) + }) + + it('should list pins with any matching cid', async () => { + await addRemotePins(ipfs, SERVICE, { + a: cid1, + b: cid2, + c: cid3, + d: cid4 + }) + + const list = await all(ipfs.pin.remote.ls({ + cid: [cid1, cid3], + status: ['queued', 'pinned', 'pinning', 'failed'], + service: SERVICE + })) + + expect(list.sort(byCID)).to.deep.equal([ + { + status: 'queued', + cid: cid1, + name: 'a' + }, + { + status: 'queued', + cid: cid3, + name: 'c' + } + ].sort(byCID)) + }) + + it('should list pins with matching cid+status', async () => { + await addRemotePins(ipfs, SERVICE, { + 'pinned-a': cid1, + 'failed-b': cid2, + 'pinned-c': cid3, + d: cid4 + }) + + const list = await all(ipfs.pin.remote.ls({ + cid: [cid1, cid2], + status: ['pinned', 'failed'], + service: SERVICE + })) + + expect(list.sort(byCID)).to.deep.equal([ + { + status: 'pinned', + cid: cid1, + name: 'pinned-a' + }, + { + status: 'failed', + cid: cid2, + name: 'failed-b' + } + ].sort(byCID)) + }) + + it('should list pins with matching cid+status+name', async () => { + await addRemotePins(ipfs, SERVICE, { + 'pinned-a': cid1, + 'failed-b': cid2, + 'pinned-c': cid3, + d: cid4 + }) + + const list = await all(ipfs.pin.remote.ls({ + cid: [cid4, cid1, cid2], + name: 'd', + status: ['queued', 'pinned'], + service: SERVICE + })) + + expect(list).to.deep.equal([ + { + status: 'queued', + cid: cid4, + name: 'd' + } + ]) + }) + }) + + it('should respect timeout option', () => { + return testTimeout(() => all(ipfs.pin.remote.ls({ + timeout: 1, + service: SERVICE + }))) + }) + }) +} + +const byCID = (a, b) => a.cid.toString() > b.cid.toString() ? 1 : -1 diff --git a/packages/interface-ipfs-core/src/pin/remote/rm-all.js b/packages/interface-ipfs-core/src/pin/remote/rm-all.js new file mode 100644 index 0000000000..9b80f37921 --- /dev/null +++ b/packages/interface-ipfs-core/src/pin/remote/rm-all.js @@ -0,0 +1,164 @@ +/* eslint-env mocha */ +'use strict' + +const { clearRemotePins, addRemotePins, clearServices } = require('../utils') +const { getDescribe, getIt, expect } = require('../../utils/mocha') +const testTimeout = require('../../utils/test-timeout') +const CID = require('cids') +const all = require('it-all') + +/** @typedef { import("ipfsd-ctl/src/factory") } Factory */ +/** + * @param {Factory} common + * @param {Object} options + */ +module.exports = (common, options) => { + const describe = getDescribe(options) + const it = getIt(options) + + const ENDPOINT = new URL(process.env.PINNING_SERVICE_ENDPOINT || '') + const KEY = process.env.PINNING_SERVIEC_KEY + const SERVICE = 'pinbot' + + const cid1 = new CID('QmbKtKBrmeRHjNCwR4zAfCJdMVu6dgmwk9M9AE9pUM9RgG') + const cid2 = new CID('QmdFyxZXsFiP4csgfM5uPu99AvFiKH62CSPDw5TP92nr7w') + const cid3 = new CID('Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP') + const cid4 = new CID('QmY9cxiHqTFoWamkQVkpmmqzBrY3hCBEL2XNu3NtX74Fuu') + + describe('.pin.remote.rmAll()', function () { + this.timeout(50 * 1000) + + let ipfs + before(async () => { + ipfs = (await common.spawn()).api + await ipfs.pin.remote.service.add(SERVICE, { + endpoint: ENDPOINT, + key: KEY + }) + }) + after(async () => { + await clearServices(ipfs) + await common.clean() + }) + + beforeEach(async () => { + await addRemotePins(ipfs, SERVICE, { + 'queued-a': cid1, + 'pinning-b': cid2, + 'pinned-c': cid3, + 'failed-d': cid4 + }) + }) + afterEach(async () => { + await clearRemotePins(ipfs) + }) + + it('.rmAll requires service option', async () => { + const result = ipfs.pin.remote.rmAll({}) + await expect(result).to.eventually.be.rejectedWith(/service name must be passed/) + }) + + it('noop if there is no match', async () => { + await ipfs.pin.remote.rmAll({ + cid: [cid1], + status: ['pinned', 'failed'], + service: SERVICE + }) + + const list = await all(ipfs.pin.remote.ls({ + status: ['queued', 'pinning', 'pinned', 'failed'], + service: SERVICE + })) + + expect(list.sort(byCID)).to.deep.equal([ + { + cid: cid1, + status: 'queued', + name: 'queued-a' + }, + { + cid: cid2, + status: 'pinning', + name: 'pinning-b' + }, + { + cid: cid3, + status: 'pinned', + name: 'pinned-c' + }, + { + cid: cid4, + status: 'failed', + name: 'failed-d' + } + ].sort(byCID)) + }) + + it('removes matching pin', async () => { + await ipfs.pin.remote.rmAll({ + cid: [cid1], + status: ['queued', 'pinning', 'pinned', 'failed'], + service: SERVICE + }) + + const list = await all(ipfs.pin.remote.ls({ + status: ['queued', 'pinning', 'pinned', 'failed'], + service: SERVICE + })) + + expect(list.sort(byCID)).to.deep.equal([ + { + cid: cid2, + status: 'pinning', + name: 'pinning-b' + }, + { + cid: cid3, + status: 'pinned', + name: 'pinned-c' + }, + { + cid: cid4, + status: 'failed', + name: 'failed-d' + } + ].sort(byCID)) + }) + + it('removes multiple matches', async () => { + const result = ipfs.pin.remote.rmAll({ + cid: [cid1, cid2], + status: ['queued', 'pinning', 'pinned', 'failed'], + service: SERVICE + }) + await expect(result).to.eventually.be.equal(undefined) + + const list = await all(ipfs.pin.remote.ls({ + status: ['queued', 'pinning', 'pinned', 'failed'], + service: SERVICE + })) + + expect(list.sort(byCID)).to.deep.equal([ + { + cid: cid3, + status: 'pinned', + name: 'pinned-c' + }, + { + cid: cid4, + status: 'failed', + name: 'failed-d' + } + ].sort(byCID)) + }) + + it('should respect timeout option', () => { + return testTimeout(() => ipfs.pin.remote.rmAll({ + timeout: 1, + service: SERVICE + })) + }) + }) +} + +const byCID = (a, b) => a.cid.toString() > b.cid.toString() ? 1 : -1 diff --git a/packages/interface-ipfs-core/src/pin/remote/rm.js b/packages/interface-ipfs-core/src/pin/remote/rm.js new file mode 100644 index 0000000000..4dead16588 --- /dev/null +++ b/packages/interface-ipfs-core/src/pin/remote/rm.js @@ -0,0 +1,182 @@ +/* eslint-env mocha */ +'use strict' + +const { clearRemotePins, addRemotePins, clearServices } = require('../utils') +const { getDescribe, getIt, expect } = require('../../utils/mocha') +const testTimeout = require('../../utils/test-timeout') +const CID = require('cids') +const all = require('it-all') + +/** @typedef { import("ipfsd-ctl/src/factory") } Factory */ +/** + * @param {Factory} common + * @param {Object} options + */ +module.exports = (common, options) => { + const describe = getDescribe(options) + const it = getIt(options) + + const ENDPOINT = new URL(process.env.PINNING_SERVICE_ENDPOINT || '') + const KEY = process.env.PINNING_SERVIEC_KEY + const SERVICE = 'pinbot' + + const cid1 = new CID('QmbKtKBrmeRHjNCwR4zAfCJdMVu6dgmwk9M9AE9pUM9RgG') + const cid2 = new CID('QmdFyxZXsFiP4csgfM5uPu99AvFiKH62CSPDw5TP92nr7w') + const cid3 = new CID('Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP') + const cid4 = new CID('QmY9cxiHqTFoWamkQVkpmmqzBrY3hCBEL2XNu3NtX74Fuu') + + describe('.pin.remote.rm()', function () { + this.timeout(50 * 1000) + + let ipfs + before(async () => { + ipfs = (await common.spawn()).api + await ipfs.pin.remote.service.add(SERVICE, { + endpoint: ENDPOINT, + key: KEY + }) + }) + after(async () => { + await clearServices(ipfs) + await common.clean() + }) + + beforeEach(async () => { + await addRemotePins(ipfs, SERVICE, { + 'queued-a': cid1, + 'pinning-b': cid2, + 'pinned-c': cid3, + 'failed-d': cid4 + }) + }) + afterEach(async () => { + await clearRemotePins(ipfs) + }) + + it('.rm requires service option', async () => { + const result = ipfs.pin.remote.rm({}) + await expect(result).to.eventually.be.rejectedWith(/service name must be passed/) + }) + + it('.rmAll requires service option', async () => { + const result = ipfs.pin.remote.rmAll({}) + await expect(result).to.eventually.be.rejectedWith(/service name must be passed/) + }) + + it('noop if there is no match', async () => { + await ipfs.pin.remote.rm({ + cid: [cid1], + status: ['pinned', 'failed'], + service: SERVICE + }) + + const list = await all(ipfs.pin.remote.ls({ + status: ['queued', 'pinning', 'pinned', 'failed'], + service: SERVICE + })) + + expect(list.sort(byCID)).to.deep.equal([ + { + cid: cid1, + status: 'queued', + name: 'queued-a' + }, + { + cid: cid2, + status: 'pinning', + name: 'pinning-b' + }, + { + cid: cid3, + status: 'pinned', + name: 'pinned-c' + }, + { + cid: cid4, + status: 'failed', + name: 'failed-d' + } + ].sort(byCID)) + }) + + it('removes matching pin', async () => { + await ipfs.pin.remote.rm({ + cid: [cid1], + status: ['queued', 'pinning', 'pinned', 'failed'], + service: SERVICE + }) + + const list = await all(ipfs.pin.remote.ls({ + status: ['queued', 'pinning', 'pinned', 'failed'], + service: SERVICE + })) + + expect(list.sort(byCID)).to.deep.equal([ + { + cid: cid2, + status: 'pinning', + name: 'pinning-b' + }, + { + cid: cid3, + status: 'pinned', + name: 'pinned-c' + }, + { + cid: cid4, + status: 'failed', + name: 'failed-d' + } + ].sort(byCID)) + }) + + it('fails on multiple matches', async () => { + const result = ipfs.pin.remote.rm({ + cid: [cid1, cid2], + status: ['queued', 'pinning', 'pinned', 'failed'], + service: SERVICE + }) + + await expect(result).to.eventually.be.rejectedWith( + /multiple remote pins are matching this query/ + ) + + const list = await all(ipfs.pin.remote.ls({ + status: ['queued', 'pinning', 'pinned', 'failed'], + service: SERVICE + })) + + expect(list.sort(byCID)).to.deep.equal([ + { + cid: cid1, + status: 'queued', + name: 'queued-a' + }, + { + cid: cid2, + status: 'pinning', + name: 'pinning-b' + }, + { + cid: cid3, + status: 'pinned', + name: 'pinned-c' + }, + { + cid: cid4, + status: 'failed', + name: 'failed-d' + } + ].sort(byCID)) + }) + + it('should respect timeout option', () => { + return testTimeout(() => ipfs.pin.remote.rm({ + timeout: 1, + service: SERVICE + })) + }) + }) +} + +const byCID = (a, b) => a.cid.toString() > b.cid.toString() ? 1 : -1 diff --git a/packages/interface-ipfs-core/src/pin/remote/service.js b/packages/interface-ipfs-core/src/pin/remote/service.js new file mode 100644 index 0000000000..b0721950a2 --- /dev/null +++ b/packages/interface-ipfs-core/src/pin/remote/service.js @@ -0,0 +1,208 @@ +/* eslint-env mocha */ +'use strict' + +const { clearServices } = require('../utils') +const { getDescribe, getIt, expect } = require('../../utils/mocha') + +/** @typedef { import("ipfsd-ctl/src/factory") } Factory */ +/** + * @param {Factory} common + * @param {Object} options + */ +module.exports = (common, options) => { + const describe = getDescribe(options) + const it = getIt(options) + + const ENDPOINT = new URL(process.env.PINNING_SERVICE_ENDPOINT || '') + const KEY = process.env.PINNING_SERVIEC_KEY + + describe('.pin.remote.service', function () { + this.timeout(50 * 1000) + + let ipfs + before(async () => { + ipfs = (await common.spawn()).api + }) + + after(async () => { + await common.clean() + }) + afterEach(() => clearServices(ipfs)) + + describe('.pin.remote.service.add', () => { + it('should add a service', async () => { + await ipfs.pin.remote.service.add('pinbot', { + endpoint: ENDPOINT, + key: KEY + }) + + const services = await ipfs.pin.remote.service.ls() + expect(services).to.deep.equal([{ + service: 'pinbot', + endpoint: ENDPOINT + }]) + }) + + it('service add requires endpoint', async () => { + const result = ipfs.pin.remote.service.add('noend', { key: 'token' }) + await expect(result).to.eventually.be.rejectedWith(/is required/) + }) + + it('service add requires key', async () => { + const result = ipfs.pin.remote.service.add('nokey', { + endpoint: ENDPOINT + }) + + await expect(result).to.eventually.be.rejectedWith(/is required/) + }) + + it('add multiple services', async () => { + await ipfs.pin.remote.service.add('pinbot', { + endpoint: ENDPOINT, + key: KEY + }) + + await ipfs.pin.remote.service.add('pinata', { + endpoint: new URL('https://api.pinata.cloud'), + key: 'somekey' + }) + + const services = await ipfs.pin.remote.service.ls() + expect(services.sort(byName)).to.deep.equal([ + { + service: 'pinbot', + endpoint: ENDPOINT + }, + { + service: 'pinata', + endpoint: new URL('https://api.pinata.cloud') + } + ].sort(byName)) + }) + + it('can not add service with existing name', async () => { + await ipfs.pin.remote.service.add('pinbot', { + endpoint: ENDPOINT, + key: KEY + }) + + const result = ipfs.pin.remote.service.add('pinbot', { + endpoint: 'http://pinbot.io/', + key: KEY + }) + + await expect(result).to.eventually.be.rejectedWith(/service already present/) + }) + }) + + describe('.pin.remote.service.ls', () => { + it('should list services', async () => { + const services = await ipfs.pin.remote.service.ls() + expect(services).to.deep.equal([]) + }) + + it('should list added service', async () => { + await ipfs.pin.remote.service.add('pinbot', { + endpoint: ENDPOINT, + key: KEY + }) + + const services = await ipfs.pin.remote.service.ls() + expect(services).to.deep.equal([{ + service: 'pinbot', + endpoint: ENDPOINT + }]) + }) + + it('should include service stats', async () => { + await ipfs.pin.remote.service.add('pinbot', { + endpoint: ENDPOINT, + key: KEY + }) + + const services = await ipfs.pin.remote.service.ls({ stat: true }) + + expect(services).to.deep.equal([{ + service: 'pinbot', + endpoint: ENDPOINT, + stat: { + status: 'valid', + pinCount: { + queued: 0, + pinning: 0, + pinned: 0, + failed: 0 + } + } + }]) + }) + + it('should report unreachable services', async () => { + await ipfs.pin.remote.service.add('pinbot', { + endpoint: ENDPOINT, + key: KEY + }) + await ipfs.pin.remote.service.add('boombot', { + endpoint: 'http://127.0.0.1:5555', + key: 'boom' + }) + + const services = await ipfs.pin.remote.service.ls({ stat: true }) + + expect(services.sort(byName)).to.deep.equal([ + { + service: 'pinbot', + endpoint: ENDPOINT, + stat: { + status: 'valid', + pinCount: { + queued: 0, + pinning: 0, + pinned: 0, + failed: 0 + } + } + }, + { + service: 'boombot', + endpoint: new URL('http://127.0.0.1:5555'), + stat: { + status: 'invalid' + } + } + ].sort(byName)) + }) + }) + + describe('.pin.remote.service.rm', () => { + it('should remove service', async () => { + await ipfs.pin.remote.service.add('pinbot', { + endpoint: ENDPOINT, + key: KEY + }) + + const services = await ipfs.pin.remote.service.ls() + expect(services).to.deep.equal([{ + service: 'pinbot', + endpoint: ENDPOINT + }]) + + await ipfs.pin.remote.service.rm('pinbot') + + expect(await ipfs.pin.remote.service.ls()).to.deep.equal([]) + }) + + it('should not fail if service does not registered', async () => { + expect(await ipfs.pin.remote.service.ls()).to.deep.equal([]) + expect(await ipfs.pin.remote.service.rm('pinbot')).to.equal(undefined) + }) + + it('expects service name', async () => { + const result = ipfs.pin.remote.service.rm() + await expect(result).to.eventually.be.rejectedWith(/is required/) + }) + }) + }) +} + +const byName = (a, b) => a.service > b.service ? 1 : -1 diff --git a/packages/interface-ipfs-core/src/pin/utils.js b/packages/interface-ipfs-core/src/pin/utils.js index 8935cb6bd7..faf17c706f 100644 --- a/packages/interface-ipfs-core/src/pin/utils.js +++ b/packages/interface-ipfs-core/src/pin/utils.js @@ -42,6 +42,41 @@ const clearPins = async (ipfs) => { await drain(ipfs.pin.rmAll(map(ipfs.pin.ls({ type: pinTypes.direct }), ({ cid }) => cid))) } +const clearRemotePins = async (ipfs) => { + for (const { service } of await ipfs.pin.remote.service.ls()) { + const cids = [] + const status = ['queued', 'pinning', 'pinned', 'failed'] + for await (const pin of ipfs.pin.remote.ls({ status, service })) { + cids.push(pin.cid) + } + + if (cids.length > 0) { + await ipfs.pin.remote.rmAll({ + cid: cids, + status, + service + }) + } + } +} + +const addRemotePins = async (ipfs, service, pins) => { + const requests = [] + for (const [name, cid] of Object.entries(pins)) { + requests.push(ipfs.pin.remote.add(cid, { + name, + service, + background: true + })) + } + await Promise.all(requests) +} + +const clearServices = async (ipfs) => { + const services = await ipfs.pin.remote.service.ls() + await Promise.all(services.map(({ service }) => ipfs.pin.remote.service.rm(service))) +} + const expectPinned = async (ipfs, cid, type = pinTypes.all, pinned = true) => { if (typeof type === 'boolean') { pinned = type @@ -70,6 +105,9 @@ async function isPinnedWithType (ipfs, cid, type) { module.exports = { fixtures, clearPins, + clearServices, + clearRemotePins, + addRemotePins, expectPinned, expectNotPinned, isPinnedWithType, diff --git a/packages/ipfs-core-types/src/pin/remote.ts b/packages/ipfs-core-types/src/pin/remote.ts new file mode 100644 index 0000000000..c148a42fb3 --- /dev/null +++ b/packages/ipfs-core-types/src/pin/remote.ts @@ -0,0 +1,95 @@ +import CID from 'cids' +import Multiaddr from 'multiaddr' +import { API as Service } from './remote/service' +import { AbortOptions } from '../basic' + +export interface API { + /** + * API for configuring remote pinning services. + */ + service: Service + + /** + * Pin a content with a given CID to a remote pinning service. + */ + add(cid:CID, options:AddOptions & AbortOptions):Promise + + /** + * Returns a list of matching pins on the remote pinning service. + */ + ls(query: Query & AbortOptions): AsyncIterable + + /** + * Removes a single pin object matching query allowing it to be garbage + * collected (if needed). Will error if multiple pins mtach provided + * query. To remove all matches use `rmAll` instead. + */ + rm(query: Query & AbortOptions): Promise + + /** + * Removes all pin object that match given query allowing them to be garbage + * collected if needed. + */ + rmAll(query: Query & AbortOptions): Promise +} + +export interface AddOptions extends RemoteServiceOptions { + /** + * Optional name for pinned data; can be used for lookups later (max 255 + * characters) + */ + name?: string + + /** + * Optional list of multiaddrs known to provide the data (max 20). + */ + origins?: Multiaddr[] + + /** + * If true, will add to the queue on the remote service and return + * immediately. If false or omitted will wait until pinned on the + * remote service. + */ + background?: boolean +} + +/** + * Reperesents query for matching pin objects. + */ +export interface Query extends RemoteServiceOptions { + /** + * If provided, will only include pin objects that have a CID from the given + * set. + */ + cid?: CID[] + /** + * If passed, will only include pin objects with names that have this name + * (case-sensitive, exact match). + */ + name?: string + + /** + * Return pin objects for pins that have one of the specified status values. + * If omitted treated as ["pinned"] + */ + status?: Status[] +} + +export interface RemoteServiceOptions { + /** + * Name of the remote pinning service to use. + */ + service: string +} + +export interface Pin { + status: Status + cid: CID + name: string +} + +export type Status = + | 'queued' + | 'pinning' + | 'pinned' + | 'failed' diff --git a/packages/ipfs-core-types/src/pin/remote/service.ts b/packages/ipfs-core-types/src/pin/remote/service.ts new file mode 100644 index 0000000000..8885219bba --- /dev/null +++ b/packages/ipfs-core-types/src/pin/remote/service.ts @@ -0,0 +1,70 @@ +import { AbortOptions } from '../../basic' + +export interface API { + /** + * Registers remote pinning service with a given name. Errors if service + * with the given name is already registered. + */ + add(name: string, credentials:Credentials & AbortOptions): Promise + + /** + * Unregisteres remote pinning service with a given name. If service with such + * name isn't registerede this is a noop. + */ + rm(name: string, options?:AbortOptions):Promise + + /** + * List registered remote pinning services. + */ + ls(options: { stat: true } & AbortOptions): Promise + ls(options?: AbortOptions):Promise +} + +export interface Credentials { + /** + * Service endpoint + */ + endpoint: URL + /** + * Service key + */ + key: string +} + +export interface RemotePinService { + /** + * Service name + */ + service: string + /** + * Service endpoint URL + */ + endpoint: URL +} + +export interface RemotePinServiceWithStat extends RemotePinService { + /** + * Pin count on the remote service. It is fetched from the remote service and + * is done only if `pinCount` option is used. Furthermore it may not be + * present if service was unreachable. + */ + stat: Stat +} + +export type Stat = ValidStat | InvalidStat + +type ValidStat = { + status: 'valid' + pinCount: PinCount +} + +type InvalidStat = { + status: 'invalid' + pinCount?: void +} +export type PinCount = { + queued: number, + pinning: number, + pinned: number, + failed: number +} diff --git a/packages/ipfs-core/package.json b/packages/ipfs-core/package.json index 786f635c18..72532b5861 100644 --- a/packages/ipfs-core/package.json +++ b/packages/ipfs-core/package.json @@ -120,7 +120,7 @@ "devDependencies": { "aegir": "^29.2.2", "delay": "^4.4.0", - "go-ipfs": "^0.7.0", + "go-ipfs": "0.8.0-rc2", "interface-ipfs-core": "^0.143.1", "ipfsd-ctl": "^7.2.0", "ipld-git": "^0.6.1", diff --git a/packages/ipfs-core/src/components/libp2p.js b/packages/ipfs-core/src/components/libp2p.js index 1bc1b0651c..5073f8e649 100644 --- a/packages/ipfs-core/src/components/libp2p.js +++ b/packages/ipfs-core/src/components/libp2p.js @@ -173,6 +173,6 @@ function getLibp2pOptions ({ options, config, datastore, keys, keychainConfig, p * @typedef {import('.').PeerId} PeerId * @typedef {import('.').Options} IPFSOptions * @typedef {import('libp2p')} LibP2P - * @typedef {import('libp2p').Libp2pOptions} Options + * @typedef {import('libp2p').Libp2pOptions & import('libp2p').constructorOptions} Options * @typedef {import('.').IPFSConfig} IPFSConfig */ diff --git a/packages/ipfs-grpc-server/src/endpoints/add.js b/packages/ipfs-grpc-server/src/endpoints/add.js index be75509981..c31a8d3b1a 100644 --- a/packages/ipfs-grpc-server/src/endpoints/add.js +++ b/packages/ipfs-grpc-server/src/endpoints/add.js @@ -56,7 +56,6 @@ module.exports = function grpcAdd (ipfs, options = {}) { if (!stream) { // start of new file - // @ts-ignore stream = streams[index] = pushable() fileInputStream.push({ diff --git a/packages/ipfs-http-client/package.json b/packages/ipfs-http-client/package.json index 829bbbc2a6..b20586ac1e 100644 --- a/packages/ipfs-http-client/package.json +++ b/packages/ipfs-http-client/package.json @@ -82,7 +82,7 @@ "devDependencies": { "aegir": "^29.2.2", "delay": "^4.4.0", - "go-ipfs": "^0.7.0", + "go-ipfs": "0.8.0-rc2", "ipfs-core": "^0.4.2", "ipfsd-ctl": "^7.2.0", "it-all": "^1.0.4", diff --git a/packages/ipfs-http-client/src/pin/index.js b/packages/ipfs-http-client/src/pin/index.js index 39f7b5ecc8..e27a8af501 100644 --- a/packages/ipfs-http-client/src/pin/index.js +++ b/packages/ipfs-http-client/src/pin/index.js @@ -1,9 +1,12 @@ 'use strict' +const Remote = require('./remote') + module.exports = config => ({ add: require('./add')(config), addAll: require('./add-all')(config), ls: require('./ls')(config), rm: require('./rm')(config), - rmAll: require('./rm-all')(config) + rmAll: require('./rm-all')(config), + remote: new Remote(config) }) diff --git a/packages/ipfs-http-client/src/pin/remote/index.js b/packages/ipfs-http-client/src/pin/remote/index.js new file mode 100644 index 0000000000..0019020f6b --- /dev/null +++ b/packages/ipfs-http-client/src/pin/remote/index.js @@ -0,0 +1,208 @@ +'use strict' + +const CID = require('cids') +const Client = require('../../lib/core') +const Service = require('./service') +const toUrlSearchParams = require('../../lib/to-url-search-params') + +/** + * @typedef {import('../..').HttpOptions} HttpOptions + * @typedef {import('../../lib/core').ClientOptions} ClientOptions + * @typedef {import('ipfs-core-types/src/basic').AbortOptions} AbortOptions + * @typedef {import('ipfs-core-types/src/pin/remote').API} API + * @typedef {import('ipfs-core-types/src/pin/remote').Pin} Pin + * @typedef {import('ipfs-core-types/src/pin/remote').AddOptions} AddOptions + * @typedef {import('ipfs-core-types/src/pin/remote').Query} Query + * @typedef {import('ipfs-core-types/src/pin/remote').Status} Status + * + * @implements {API} + */ +class Remote { + /** + * @param {ClientOptions} options + */ + constructor (options) { + /** @private */ + this.client = new Client(options) + /** @readonly */ + this.service = new Service(options) + } + + /** + * Stores an IPFS object(s) from a given path to a remote pinning service. + * + * @param {CID} cid + * @param {AddOptions & AbortOptions & HttpOptions} options + * @returns {Promise} + */ + add (cid, options) { + return Remote.add(this.client, cid, options) + } + + /** + * @param {Client} client + * @param {CID} cid + * @param {AddOptions & AbortOptions & HttpOptions} options + */ + static async add (client, cid, { timeout, signal, headers, ...options }) { + const response = await client.post('pin/remote/add', { + timeout, + signal, + headers, + searchParams: encodeAddParams({ cid, ...options }) + }) + + return Remote.decodePin(await response.json()) + } + + /** + * @param {Object} json + * @param {string} json.Name + * @param {string} json.Cid + * @param {Status} json.Status + * @returns {Pin} + */ + static decodePin ({ Name: name, Status: status, Cid: cid }) { + return { + cid: new CID(cid), + name, + status + } + } + + /** + * Returns a list of matching pins on the remote pinning service. + * + * @param {Query & AbortOptions & HttpOptions} query + */ + ls (query) { + return Remote.ls(this.client, query) + } + + /** + * + * @param {Client} client + * @param {Query & AbortOptions & HttpOptions} options + * @returns {AsyncIterable} + */ + static async * ls (client, { timeout, signal, headers, ...query }) { + const response = await client.post('pin/remote/ls', { + signal, + timeout, + headers, + searchParams: encodeQuery(query) + }) + + for await (const pin of response.ndjson()) { + yield Remote.decodePin(pin) + } + } + + /** + * Removes a single pin object matching query allowing it to be garbage + * collected (if needed). Will error if multiple pins mtach provided + * query. To remove all matches use `rmAll` instead. + * + * @param {Query & AbortOptions & HttpOptions} query + */ + rm (query) { + return Remote.rm(this.client, { ...query, all: false }) + } + + /** + * Removes all pin object that match given query allowing them to be garbage + * collected if needed. + * + * @param {Query & AbortOptions & HttpOptions} query + */ + rmAll (query) { + return Remote.rm(this.client, { ...query, all: true }) + } + + /** + * + * @param {Client} client + * @param {{all: boolean} & Query & AbortOptions & HttpOptions} options + */ + static async rm (client, { timeout, signal, headers, ...query }) { + await client.post('pin/remote/rm', { + timeout, + signal, + headers, + searchParams: encodeQuery(query) + }) + } +} + +/** + * @param {any} service + * @returns {string} + */ +const encodeService = (service) => { + if (typeof service === 'string' && service !== '') { + return service + } else { + throw new TypeError('service name must be passed') + } +} + +/** + * @param {any} cid + * @returns {string} + */ +const encodeCID = (cid) => { + if (CID.isCID(cid)) { + return cid.toString() + } else { + throw new TypeError(`CID instance expected instead of ${cid}`) + } +} + +/** + * @param {Query & { all?: boolean }} query + * @returns {URLSearchParams} + */ +const encodeQuery = ({ service, cid, name, status, all }) => { + const query = toUrlSearchParams({ + service: encodeService(service), + name, + force: all ? true : undefined + }) + + if (cid) { + for (const value of cid) { + query.append('cid', encodeCID(value)) + } + } + + if (status) { + for (const value of status) { + query.append('status', value) + } + } + + return query +} + +/** + * @param {AddOptions & {cid:CID}} options + * @returns {URLSearchParams} + */ +const encodeAddParams = ({ cid, service, background, name, origins }) => { + const params = toUrlSearchParams({ + arg: encodeCID(cid), + service: encodeService(service), + name, + background: background ? true : undefined + }) + + if (origins) { + for (const origin of origins) { + params.append('origin', origin.toString()) + } + } + + return params +} + +module.exports = Remote diff --git a/packages/ipfs-http-client/src/pin/remote/service.js b/packages/ipfs-http-client/src/pin/remote/service.js new file mode 100644 index 0000000000..9a718e8475 --- /dev/null +++ b/packages/ipfs-http-client/src/pin/remote/service.js @@ -0,0 +1,162 @@ +'use strict' + +const Client = require('../../lib/core') +const toUrlSearchParams = require('../../lib/to-url-search-params') + +/** + * @typedef {import('../../lib/core').ClientOptions} ClientOptions + * @typedef {import('../..').HttpOptions} HttpOptions + * @typedef {import('ipfs-core-types/src/basic').AbortOptions} AbortOptions + * @typedef {import('ipfs-core-types/src/pin/remote/service').API} API + * @typedef {import('ipfs-core-types/src/pin/remote/service').Credentials} Credentials + * @typedef {import('ipfs-core-types/src/pin/remote/service').RemotePinService} RemotePinService + * @typedef {import('ipfs-core-types/src/pin/remote/service').RemotePinServiceWithStat} RemotePinServiceWithStat + * @implements {API} + */ +class Service { + /** + * @param {ClientOptions} options + */ + constructor (options) { + /** @private */ + this.client = new Client(options) + } + + /** + * @param {Client} client + * @param {string} name + * @param {Credentials & AbortOptions & HttpOptions} options + */ + static async add (client, name, options) { + const { endpoint, key, headers, timeout, signal } = options + await client.post('pin/remote/service/add', { + timeout, + signal, + searchParams: toUrlSearchParams({ + arg: [name, Service.encodeEndpoint(endpoint), key] + }), + headers + }) + } + + /** + * @param {URL} url + */ + static encodeEndpoint (url) { + const href = String(url) + if (href === 'undefined') { + throw Error('endpoint is required') + } + // Workaround trailing `/` issue in go-ipfs + // @see https://github.com/ipfs/go-ipfs/issues/7826 + return href[href.length - 1] === '/' ? href.slice(0, -1) : href + } + + /** + * @param {Client} client + * @param {string} name + * @param {AbortOptions & HttpOptions} [options] + */ + static async rm (client, name, { timeout, signal, headers } = {}) { + await client.post('pin/remote/service/rm', { + timeout, + signal, + headers, + searchParams: toUrlSearchParams({ + arg: name + }) + }) + } + + /** + * @template {true} Stat + * @param {Client} client + * @param {{ stat?: Stat } & AbortOptions & HttpOptions} [options] + */ + static async ls (client, { stat, timeout, signal, headers } = {}) { + const response = await client.post('pin/remote/service/ls', { + searchParams: stat === true ? toUrlSearchParams({ stat }) : undefined, + timeout, + signal, + headers + }) + + /** @type {{RemoteServices: Object[]}} */ + const { RemoteServices } = await response.json() + + /** @type {Stat extends true ? RemotePinServiceWithStat[] : RemotePinService []} */ + return (RemoteServices.map(Service.decodeRemoteService)) + } + + /** + * @param {Object} json + * @returns {RemotePinServiceWithStat} + */ + static decodeRemoteService (json) { + return { + service: json.Service, + endpoint: new URL(json.ApiEndpoint), + ...(json.Stat && { stat: Service.decodeStat(json.Stat) }) + } + } + + /** + * @param {Object} json + * @returns {import('ipfs-core-types/src/pin/remote/service').Stat} + */ + static decodeStat (json) { + switch (json.Status) { + case 'valid': { + const { Pinning, Pinned, Queued, Failed } = json.PinCount + return { + status: 'valid', + pinCount: { + queued: Queued, + pinning: Pinning, + pinned: Pinned, + failed: Failed + } + } + } + case 'invalid': { + return { status: 'invalid' } + } + default: { + return { status: json.Status } + } + } + } + + /** + * Registers remote pinning service with a given name. Errors if service + * with the given name is already registered. + * + * @param {string} name + * @param {Credentials & AbortOptions & HttpOptions} options + */ + add (name, options) { + return Service.add(this.client, name, options) + } + + /** + * Unregisteres remote pinning service with a given name. If service with such + * name isn't registerede this is a noop. + * + * @param {string} name + * @param {AbortOptions & HttpOptions} [options] + */ + rm (name, options) { + return Service.rm(this.client, name, options) + } + + /** + * List registered remote pinning services. + * + * @param {{ stat?: true } & AbortOptions & HttpOptions} [options] + */ + ls (options) { + return Service.ls(this.client, options) + } +} + +module.exports = Service diff --git a/packages/ipfs/.aegir.js b/packages/ipfs/.aegir.js index e1b92f5939..afe6b207a7 100644 --- a/packages/ipfs/.aegir.js +++ b/packages/ipfs/.aegir.js @@ -2,11 +2,13 @@ const { createServer } = require('ipfsd-ctl') const MockPreloadNode = require('./test/utils/mock-preload-node') +const PinningService = require('./test/utils/mock-pinning-service') const EchoServer = require('aegir/utils/echo-server') const webRTCStarSigServer = require('libp2p-webrtc-star/src/sig-server') const path = require('path') let preloadNode +let pinningService let echoServer = new EchoServer() // the second signalling server is needed for the interface test 'should list peers only once even if they have multiple addresses' @@ -38,23 +40,28 @@ module.exports = { node: { pre: async () => { preloadNode = MockPreloadNode.createNode() + pinningService = await PinningService.start() await preloadNode.start(), await echoServer.start() return { env: { + PINNING_SERVICE_ENDPOINT: pinningService.endpoint, + PINNING_SERVIEC_KEY: pinningService.token, ECHO_SERVER: `http://${echoServer.host}:${echoServer.port}` } } }, post: async () => { - await preloadNode.stop(), + await preloadNode.stop() + await PinningService.stop(pinningService) await echoServer.stop() } }, browser: { pre: async () => { preloadNode = MockPreloadNode.createNode() + pinningService = await PinningService.start() await preloadNode.start() await echoServer.start() @@ -94,6 +101,8 @@ module.exports = { return { env: { + PINNING_SERVICE_ENDPOINT: pinningService.endpoint, + PINNING_SERVIEC_KEY: pinningService.token, ECHO_SERVER: `http://${echoServer.host}:${echoServer.port}` } } @@ -101,6 +110,7 @@ module.exports = { post: async () => { await ipfsdServer.stop() await preloadNode.stop() + await PinningService.stop(pinningService) await echoServer.stop() await sigServerA.stop() await sigServerB.stop() diff --git a/packages/ipfs/package.json b/packages/ipfs/package.json index 7a6bcaa1f9..27ba8bbd5c 100644 --- a/packages/ipfs/package.json +++ b/packages/ipfs/package.json @@ -49,7 +49,7 @@ "aegir": "^29.2.2", "cross-env": "^7.0.0", "electron-webrtc": "^0.3.0", - "go-ipfs": "^0.7.0", + "go-ipfs": "0.8.0-rc2", "interface-ipfs-core": "^0.143.1", "ipfs-client": "^0.2.2", "ipfs-http-client": "^48.2.2", @@ -61,7 +61,8 @@ "merge-options": "^2.0.0", "rimraf": "^3.0.2", "typescript": "4.0.x", - "wrtc": "^0.4.6" + "wrtc": "^0.4.6", + "mock-ipfs-pinning-service": "^0.1.2" }, "typesVersions": { "*": { diff --git a/packages/ipfs/test/interface-http-go.js b/packages/ipfs/test/interface-http-go.js index 565875f09a..e381690195 100644 --- a/packages/ipfs/test/interface-http-go.js +++ b/packages/ipfs/test/interface-http-go.js @@ -586,6 +586,8 @@ describe('interface-ipfs-core over ipfs-http-client tests against go-ipfs', () = ] }) + tests.pin.remote(commonFactory) + tests.ping(commonFactory, { skip: [ { diff --git a/packages/ipfs/test/utils/mock-pinning-service.js b/packages/ipfs/test/utils/mock-pinning-service.js new file mode 100644 index 0000000000..fae4c4f9f3 --- /dev/null +++ b/packages/ipfs/test/utils/mock-pinning-service.js @@ -0,0 +1,52 @@ +'use strict' + +const http = require('http') +const { setup } = require('mock-ipfs-pinning-service') + +const defaultPort = 1139 +const defaultToken = 'secret' + +class PinningService { + /** + * @param {Object} options + * @param {number} [options.port] + * @param {string|null} [options.token] + * @returns {Promise} + */ + static async start ({ port = defaultPort, token = defaultToken } = {}) { + const service = await setup({ token }) + const server = http.createServer(service) + await new Promise(resolve => server.listen(port, resolve)) + + return new PinningService({ server, host: '127.0.0.1', port, token }) + } + + /** + * @param {PinningService} service + * @returns {Promise} + */ + static stop (service) { + return new Promise((resolve, reject) => { + service.server.close((error) => { + if (error) { + reject(error) + } else { + resolve() + } + }) + }) + } + + constructor ({ server, host, port, token }) { + this.server = server + this.host = host + this.port = port + this.token = token + } + + get endpoint () { + return `http://${this.host}:${this.port}` + } +} + +module.exports = PinningService