feat: store pins in datastore instead of a DAG

docs/core-api/PIN.md

@@ -1,30 +1,40 @@
 # Pin API <!-- omit in toc -->
 
-- [`ipfs.pin.add(cid, [options])`](#ipfspinaddcid-options)
+- [`ipfs.pin.add(ipfsPath, [options])`](#ipfspinaddipfspath-options)
   - [Parameters](#parameters)
   - [Options](#options)
   - [Returns](#returns)
   - [Example](#example)
-- [`ipfs.pin.ls([options])`](#ipfspinlsoptions)
+- [`ipfs.pin.addAll(source, [options])`](#ipfspinaddallsource-options)
   - [Parameters](#parameters-1)
   - [Options](#options-1)
   - [Returns](#returns-1)
   - [Example](#example-1)
-- [`ipfs.pin.rm(cid, [options])`](#ipfspinrmcid-options)
+- [`ipfs.pin.ls([options])`](#ipfspinlsoptions)
   - [Parameters](#parameters-2)
   - [Options](#options-2)
   - [Returns](#returns-2)
   - [Example](#example-2)
-
-## `ipfs.pin.add(cid, [options])`
+- [`ipfs.pin.rm(ipfsPath, [options])`](#ipfspinrmipfspath-options)
+  - [Parameters](#parameters-3)
+  - [Options](#options-3)
+  - [Returns](#returns-3)
+  - [Example](#example-3)
+- [`ipfs.pin.rmAll(source, [options])`](#ipfspinrmallsource-options)
+  - [Parameters](#parameters-4)
+  - [Options](#options-4)
+  - [Returns](#returns-4)
+  - [Example](#example-4)
+
+## `ipfs.pin.add(ipfsPath, [options])`
 
 > Adds an IPFS object to the pinset and also stores it to the IPFS repo. pinset is the set of hashes currently pinned (not gc'able)
 
 ### Parameters
 
 | Name | Type | Description |
 | ---- | ---- | ----------- |
-| cid | [CID][] | Pin this CID in your repo |
+| source | [CID][] or String | A CID or IPFS Path to pin in your repo |
 
 ### Options
 
@@ -40,9 +50,45 @@ An optional object which may have the following keys:
 
 | Type | Description |
 | -------- | -------- |
-| `Promise<{ cid: CID }>` | An array of objects that represent the files that were pinned |
+| [CID[] | The CIDs that was pinned |
+
+### Example
+
+```JavaScript
+const cid of ipfs.pin.add(new CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u'))
+console.log(cid)
+// Logs:
+// CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u')
+```
+
+A great source of [examples][] can be found in the tests for this API.
+
+## `ipfs.pin.addAll(source, [options])`
+
+> Adds multiple IPFS objects to the pinset and also stores it to the IPFS repo. pinset is the set of hashes currently pinned (not gc'able)
+
+### Parameters
+
+| Name | Type | Description |
+| ---- | ---- | ----------- |
+| source | `AsyncIterable<{ cid: CID, path: String, recursive: Boolean, comments: String }>` | One or more CIDs or IPFS Paths to pin in your repo |
+
+### Options
+
+An optional object which may have the following keys:
+
+| Name | Type | Default | Description |
+| ---- | ---- | ------- | ----------- |
+| timeout | `Number` | `undefined` | A timeout in ms |
+| signal | [AbortSignal][] | `undefined` |  Can be used to cancel any long running requests started as a result of this call |
+
+### Returns
+
+| Type | Description |
+| -------- | -------- |
+| `AsyncIterable<CID>` | An async iterable that yields the CIDs that were pinned |
 
-an array of objects is returned, each of the form:
+Each yielded object has the form:
 
 ```JavaScript
 {
@@ -53,10 +99,11 @@ an array of objects is returned, each of the form:
 ### Example
 
 ```JavaScript
-const pinset = await ipfs.pin.add('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u')
-console.log(pinset)
+for await (const cid of ipfs.pin.addAll(new CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u'))) {
+  console.log(cid)
+}
 // Logs:
-// [ { cid: CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u') } ]
+// CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u')
 ```
 
 A great source of [examples][] can be found in the tests for this API.
@@ -111,15 +158,15 @@ for await (const { cid, type } of ipfs.pin.ls({
 
 A great source of [examples][] can be found in the tests for this API.
 
-## `ipfs.pin.rm(cid, [options])`
+## `ipfs.pin.rm(ipfsPath, [options])`
 
 > Unpin this block from your repo
 
 ### Parameters
 
 | Name | Type | Description |
 | ---- | ---- | ----------- |
-| cid | [CID][] | Unpin this CID |
+| ipfsPath | [CID][] of String | Unpin this CID or IPFS Path |
 
 ### Options
 
@@ -135,15 +182,52 @@ An optional object which may have the following keys:
 
 | Type | Description |
 | -------- | -------- |
-| `Promise<{ cid: CID }>` | An array of unpinned objects |
+| [CID][] | The CIDs that was unpinned |
 
 ### Example
 
 ```JavaScript
-const pinset = await ipfs.pin.rm('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u')
-console.log(pinset)
-// prints the hashes that were unpinned
-// [ { cid: CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u') } ]
+const cid of ipfs.pin.rm(new CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u'))
+console.log(cid)
+// prints the CID that was unpinned
+// CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u')
+```
+
+A great source of [examples][] can be found in the tests for this API.
+
+## `ipfs.pin.rmAll(source, [options])`
+
+> Unpin one or more blocks from your repo
+
+### Parameters
+
+| Name | Type | Description |
+| ---- | ---- | ----------- |
+| source | [CID][], String or `AsyncIterable<{ cid: CID, path: String, recursive: Boolean }>` | Unpin this CID |
+
+### Options
+
+An optional object which may have the following keys:
+
+| Name | Type | Default | Description |
+| ---- | ---- | ------- | ----------- |
+| timeout | `Number` | `undefined` | A timeout in ms |
+| signal | [AbortSignal][] | `undefined` |  Can be used to cancel any long running requests started as a result of this call |
+
+### Returns
+
+| Type | Description |
+| -------- | -------- |
+| `AsyncIterable<CID>` | An async iterable that yields the CIDs that were unpinned |
+
+### Example
+
+```JavaScript
+for await (const cid of ipfs.pin.rmAll(new CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u'))) {
+  console.log(cid)
+}
+// prints the CIDs that were unpinned
+// CID('QmWATWQ7fVPP2EFGu71UkfnqhYXDYH566qy47CnJDgvs8u')
 ```
 
 A great source of [examples][] can be found in the tests for this API.

examples/custom-ipfs-repo/package.json

@@ -12,7 +12,7 @@
   "dependencies": {
     "datastore-fs": "^1.1.0",
     "ipfs": "^0.48.0",
-    "ipfs-repo": "^4.0.0",
+    "ipfs-repo": "^5.0.0",
     "it-all": "^1.0.1"
   },
   "devDependencies": {

packages/interface-ipfs-core/package.json

@@ -51,6 +51,7 @@
     "it-concat": "^1.0.0",
     "it-drain": "^1.0.1",
     "it-last": "^1.0.1",
+    "it-map": "^1.0.2",
     "it-pushable": "^1.3.1",
     "multiaddr": "^7.4.3",
     "multibase": "^1.0.1",

packages/interface-ipfs-core/src/block/rm.js

@@ -5,6 +5,7 @@ const { Buffer } = require('buffer')
 const { getDescribe, getIt, expect } = require('../utils/mocha')
 const { nanoid } = require('nanoid')
 const all = require('it-all')
+const last = require('it-last')
 const drain = require('it-drain')
 const CID = require('cids')
 const testTimeout = require('../utils/test-timeout')
@@ -151,13 +152,12 @@ module.exports = (common, options) => {
         format: 'raw',
         hashAlg: 'sha2-256'
       })
-      await ipfs.pin.add(cid.toString())
+      await ipfs.pin.add(cid)
 
-      const result = await all(ipfs.block.rm(cid))
+      const result = await last(ipfs.block.rm(cid))
 
-      expect(result).to.be.an('array').and.to.have.lengthOf(1)
-      expect(result[0]).to.have.property('error')
-      expect(result[0].error.message).to.include('pinned')
+      expect(result).to.have.property('error').that.is.an('Error')
+        .with.property('message').that.includes('pinned')
     })
   })
 }

packages/interface-ipfs-core/src/pin/add-all.js

@@ -0,0 +1,128 @@
+/* eslint-env mocha */
+'use strict'
+
+const { fixtures, clearPins } = require('./utils')
+const { getDescribe, getIt, expect } = require('../utils/mocha')
+const all = require('it-all')
+const drain = require('it-drain')
+const testTimeout = require('../utils/test-timeout')
+const CID = require('cids')
+
+/** @typedef { import("ipfsd-ctl/src/factory") } Factory */
+/**
+ * @param {Factory} common
+ * @param {Object} options
+ */
+module.exports = (common, options) => {
+  const describe = getDescribe(options)
+  const it = getIt(options)
+
+  describe('.pin.addAll', function () {
+    this.timeout(50 * 1000)
+
+    let ipfs
+    before(async () => {
+      ipfs = (await common.spawn()).api
+
+      await drain(
+        ipfs.addAll(
+          fixtures.files.map(file => ({ content: file.data })), {
+            pin: false
+          }
+        )
+      )
+
+      await drain(
+        ipfs.addAll(fixtures.directory.files.map(
+          file => ({
+            path: file.path,
+            content: file.data
+          })
+        ), {
+          pin: false
+        })
+      )
+    })
+
+    after(() => common.clean())
+
+    beforeEach(() => {
+      return clearPins(ipfs)
+    })
+
+    async function testAddPinInput (source) {
+      const pinset = await all(ipfs.pin.addAll(source))
+
+      expect(pinset).to.have.deep.members([
+        fixtures.files[0].cid,
+        fixtures.files[1].cid
+      ])
+    }
+
+    it('should add an array of CIDs', () => {
+      return testAddPinInput([
+        fixtures.files[0].cid,
+        fixtures.files[1].cid
+      ])
+    })
+
+    it('should add a generator of CIDs', () => {
+      return testAddPinInput(function * () {
+        yield fixtures.files[0].cid
+        yield fixtures.files[1].cid
+      }())
+    })
+
+    it('should add an async generator of CIDs', () => {
+      return testAddPinInput(async function * () { // eslint-disable-line require-await
+        yield fixtures.files[0].cid
+        yield fixtures.files[1].cid
+      }())
+    })
+
+    it('should add an array of pins with options', () => {
+      return testAddPinInput([
+        {
+          cid: fixtures.files[0].cid,
+          recursive: false
+        },
+        {
+          cid: fixtures.files[1].cid,
+          recursive: true
+        }
+      ])
+    })
+
+    it('should add a generator of pins with options', () => {
+      return testAddPinInput(function * () {
+        yield {
+          cid: fixtures.files[0].cid,
+          recursive: false
+        }
+        yield {
+          cid: fixtures.files[1].cid,
+          recursive: true
+        }
+      }())
+    })
+
+    it('should add an async generator of pins with options', () => {
+      return testAddPinInput(async function * () { // eslint-disable-line require-await
+        yield {
+          cid: fixtures.files[0].cid,
+          recursive: false
+        }
+        yield {
+          cid: fixtures.files[1].cid,
+          recursive: true
+        }
+      }())
+    })
+
+    it('should respect timeout option when pinning a block', () => {
+      return testTimeout(() => ipfs.pin.addAll([new CID('Qmd7qZS4T7xXtsNFdRoK1trfMs5zU94EpokQ9WFtxdPxsZ')], {
+        timeout: 1
+      }))
+    })
+  })
+}