feat: integrate ipfs-repo-migrations tool (#2527)

Resolves: #1115

Supersedes: #2044

Depends on:

* [x] ipfs/js-ipfs-repo#202

.aegir.js

@@ -11,7 +11,7 @@ const preloadNode = MockPreloadNode.createNode()
 const echoServer = EchoServer.createServer()
 
 module.exports = {
- bundlesize: { maxSize: '685kB' },
+ bundlesize: { maxSize: '650kB' },
  webpack: {
  resolve: {
  mainFields: ['browser', 'main'],

README.md

@@ -284,6 +284,18 @@ Example:
 const node = await IPFS.create({ repo: '/var/ipfs/data' })
 ```
 
+##### `options.repoAutoMigrate`
+
+| Type | Default |
+|------|---------|
+| boolean | `true` |
+
+`js-ipfs` comes bundled with a tool that automatically migrates your IPFS repository when a new version is available.
+
+**For apps that build on top of `js-ipfs` and run in the browser environment, be aware that disabling automatic
+migrations leaves the user with no way to run the migrations because there is no CLI in the browser. In such
+a case, you should provide a way to trigger migrations manually.**
+
 ##### `options.init`
 
 | Type | Default |

package.json

@@ -104,7 +104,7 @@
  "ipfs-http-response": "~0.4.0",
  "ipfs-mfs": "^0.13.0",
  "ipfs-multipart": "^0.2.0",
- "ipfs-repo": "^0.28.1",
+ "ipfs-repo": "^0.29.0",
  "ipfs-unixfs": "~0.1.16",
  "ipfs-unixfs-exporter": "^0.38.0",
  "ipfs-unixfs-importer": "^0.40.0",

src/cli/bin.js

@@ -26,6 +26,7 @@ if (!semver.satisfies(process.versions.node, pkg.engines.node)) {
 const YargsPromise = require('yargs-promise')
 const updateNotifier = require('update-notifier')
 const debug = require('debug')('ipfs:cli')
+const { errors: { InvalidRepoVersionError } } = require('ipfs-repo')
 const parser = require('./parser')
 const commandAlias = require('./command-alias')
 const { print } = require('./utils')
@@ -50,13 +51,19 @@ cli
  })
  .catch(({ error, argv }) => {
  getIpfs = argv && argv.getIpfs
+
+ if (error.code === InvalidRepoVersionError.code) {
+ error.message = 'Incompatible repo version. Migration needed. Pass --migrate for automatic migration'
+ }
+
  if (error.message) {
  print(error.message)
  debug(error)
  } else {
  print('Unknown error, please re-run the command with DEBUG=ipfs:cli to see debug output')
  debug(error)
  }
+
  process.exit(1)
  })
  .finally(() => {

src/cli/commands/daemon.js

@@ -73,6 +73,7 @@ module.exports = {
  config,
  silent: argv.silent,
  repo: process.env.IPFS_PATH,
+ repoAutoMigrate: argv.migrate,
  offline: argv.offline,
  pass: argv.pass,
  preload: { enabled: argv.enablePreload },
@@ -96,10 +97,8 @@ module.exports = {
  print(`Web UI available at ${toUri(apiServer.info.ma)}/webui`)
  })
  } catch (err) {
- if (err.code === 'ENOENT' && err.message.match(/uninitialized/i)) {
- print('Error: no initialized ipfs repo found in ' + repoPath)
- print('please run: jsipfs init')
- process.exit(1)
+ if (err.code === 'ERR_REPO_NOT_INITIALIZED' || err.message.match(/uninitialized/i)) {
+ err.message = 'no initialized ipfs repo found in ' + repoPath + '\nplease run: jsipfs init'
  }
  throw err
  }

src/cli/daemon.js

@@ -54,17 +54,8 @@ class Daemon {
  }
 
  // start the daemon
- const ipfsOpts = Object.assign({ }, { init: true, start: true, libp2p }, this._options)
- const ipfs = new IPFS(ipfsOpts)
-
- await new Promise((resolve, reject) => {
- ipfs.once('error', err => {
- this._log('error starting core', err)
- err.code = 'ENOENT'
- reject(err)
- })
- ipfs.once('start', resolve)
- })
+ const ipfsOpts = Object.assign({}, { init: true, start: true, libp2p }, this._options)
+ const ipfs = await IPFS.create(ipfsOpts)
 
  this._ipfs = ipfs
 

src/cli/parser.js

@@ -19,6 +19,11 @@ const parser = yargs
  type: 'string',
  default: ''
  })
+ .option('migrate', {
+ desc: 'Enable/disable automatic repo migrations',
+ type: 'boolean',
+ default: false
+ })
  .epilog(utils.ipfsPathHelp)
  .demandCommand(1)
  .fail((msg, err, yargs) => {

src/cli/utils.js

@@ -4,9 +4,7 @@ const fs = require('fs')
 const os = require('os')
 const multiaddr = require('multiaddr')
 const path = require('path')
-const debug = require('debug')
-const log = debug('cli')
-log.error = debug('cli:error')
+const log = require('debug')('ipfs:cli:utils')
 const Progress = require('progress')
 const byteman = require('byteman')
 const promisify = require('promisify-es6')
@@ -47,6 +45,7 @@ exports.getIPFS = (argv, callback) => {
  const IPFS = require('../core')
  const node = new IPFS({
  silent: argv.silent,
+ repoAutoMigrate: argv.migrate,
  repo: exports.getRepoPath(),
  init: false,
  start: false,
@@ -60,7 +59,7 @@ exports.getIPFS = (argv, callback) => {
  })
 
  node.on('error', (err) => {
- throw err
+ callback(err)
  })
 
  node.once('ready', () => {

src/core/config.js

@@ -28,6 +28,7 @@ const s = superstruct({
 const configSchema = s({
  repo: optional(s('object|string')),
  repoOwner: 'boolean?',
+ repoAutoMigrate: 'boolean?',
  preload: s({
  enabled: 'boolean?',
  addresses: optional(s(['multiaddr'])),

src/core/index.js

@@ -70,7 +70,7 @@ class IPFS extends EventEmitter {
 
  if (typeof options.repo === 'string' ||
  options.repo === undefined) {
- this._repo = defaultRepo(options.repo)
+ this._repo = defaultRepo(options)
  } else {
  this._repo = options.repo
  }

src/core/runtime/repo-browser.js

@@ -2,7 +2,7 @@
 
 const IPFSRepo = require('ipfs-repo')
 
-module.exports = (dir) => {
- const repoPath = dir || 'ipfs'
- return new IPFSRepo(repoPath)
+module.exports = (options) => {
+ const repoPath = options.repo || 'ipfs'
+ return new IPFSRepo(repoPath, { autoMigrate: options.repoAutoMigrate })
 }

src/core/runtime/repo-nodejs.js

@@ -4,8 +4,8 @@ const os = require('os')
 const IPFSRepo = require('ipfs-repo')
 const path = require('path')
 
-module.exports = (dir) => {
- const repoPath = dir || path.join(os.homedir(), '.jsipfs')
+module.exports = (options) => {
+ const repoPath = options.repo || path.join(os.homedir(), '.jsipfs')
 
- return new IPFSRepo(repoPath)
+ return new IPFSRepo(repoPath, { autoMigrate: options.repoAutoMigrate })
 }

test/cli/general.js

@@ -1,8 +1,17 @@
 /* eslint-env mocha */
 'use strict'
 
+const os = require('os')
+const fs = require('fs').promises
+const path = require('path')
+const hat = require('hat')
 const { expect } = require('interface-ipfs-core/src/utils/mocha')
+const { repoVersion } = require('ipfs-repo')
+const promisify = require('promisify-es6')
+const ncp = promisify(require('ncp').ncp)
 const runOnAndOff = require('../utils/on-and-off')
+const ipfsExec = require('../utils/ipfs-exec')
+const clean = require('../utils/clean')
 
 describe('general cli options', () => runOnAndOff.off((thing) => {
  it('should handle --silent flag', async () => {
@@ -17,3 +26,83 @@ describe('general cli options', () => runOnAndOff.off((thing) => {
  expect(out).to.include('again')
  })
 }))
+
+describe('--migrate', () => {
+ let ipfs, repoPath
+
+ async function setRepoVersion (version) {
+ await fs.writeFile(path.join(repoPath, 'version'), version)
+ }
+
+ async function getRepoVersion () {
+ return parseInt(await fs.readFile(path.join(repoPath, 'version'), 'utf8'))
+ }
+
+ beforeEach(async () => {
+ repoPath = path.join(os.tmpdir(), `ipfs-${hat()}`)
+ const v7RepoPath = path.join(__dirname, '../fixtures/v7-repo')
+ await ncp(v7RepoPath, repoPath)
+ ipfs = ipfsExec(repoPath)
+ })
+
+ afterEach(() => clean(repoPath))
+
+ it('should not migrate for daemon command when --migrate flag not set', async () => {
+ // There are no migrations prior to 7 so it's safe to set version to 5 since
+ // the repo is the same. We set to 5 because version 6 & 7 are considered
+ // the same in repo.version.check.
+ await setRepoVersion(5)
+ const err = await ipfs.fail('daemon')
+ expect(err.stdout).to.include('Pass --migrate for automatic migration')
+ const version = await getRepoVersion()
+ expect(version).to.equal(5) // Should not have migrated
+ })
+
+ it('should not migrate for other commands when --migrate flag not set', async () => {
+ // There are no migrations prior to 7 so it's safe to set version to 5 since
+ // the repo is the same. We set to 5 because version 6 & 7 are considered
+ // the same in repo.version.check.
+ await setRepoVersion(5)
+ const err = await ipfs.fail('files ls')
+ expect(err.stdout).to.include('Pass --migrate for automatic migration')
+ const version = await getRepoVersion()
+ expect(version).to.equal(5) // Should not have migrated
+ })
+
+ it('should migrate for daemon command when --migrate flag set', async () => {
+ // There are no migrations prior to 7 so it's safe to set version to 5 since
+ // the repo is the same. We set to 5 because version 6 & 7 are considered
+ // the same in repo.version.check.
+ await setRepoVersion(5)
+
+ const daemon = ipfs('daemon --migrate')
+ let stdout = ''
+
+ daemon.stdout.on('data', data => {
+ stdout += data.toString('utf8')
+
+ if (stdout.includes('Daemon is ready')) {
+ daemon.kill()
+ }
+ })
+
+ await expect(daemon)
+ .to.eventually.be.rejected()
+ .and.to.include({
+ killed: true
+ })
+
+ const version = await getRepoVersion()
+ expect(version).to.equal(repoVersion) // Should have migrated to latest
+ })
+
+ it('should migrate for other commands when --migrate flag set', async () => {
+ // There are no migrations prior to 7 so it's safe to set version to 5 since
+ // the repo is the same. We set to 5 because version 6 & 7 are considered
+ // the same in repo.version.check.
+ await setRepoVersion(5)
+ await ipfs('files ls --migrate')
+ const version = await getRepoVersion()
+ expect(version).to.equal(repoVersion) // Should have migrated to latest
+ })
+})

test/fixtures/v7-repo/blocks/7J/CIQKKLBWAIBQZOIS5X7E32LQAL6236OUKZTMHPQSFIXPWXNZHQOV7JQ.data

@@ -0,0 +1,55 @@
+
+��
+ IPFS -- Inter-Planetary File system
+
+IPFS is a global, versioned, peer-to-peer filesystem. It combines good ideas
+from Git, BitTorrent, Kademlia, SFS, and the Web. It is like a single bit-
+torrent swarm, exchanging git objects. IPFS provides an interface as simple
+as the HTTP web, but with permanence built in. You can also mount the world
+at /ipfs.
+
+IPFS is a protocol:
+- defines a content-addressed file system
+- coordinates content delivery
+- combines Kademlia + BitTorrent + Git
+
+IPFS is a filesystem:
+- has directories and files
+- mountable filesystem (via FUSE)
+
+IPFS is a web:
+- can be used to view documents like the web
+- files accessible via HTTP at `http://ipfs.io/<path>`
+- browsers or extensions can learn to use `ipfs://` directly
+- hash-addressed content guarantees authenticity
+
+IPFS is modular:
+- connection layer over any network protocol
+- routing layer
+- uses a routing layer DHT (kademlia/coral)
+- uses a path-based naming service
+- uses bittorrent-inspired block exchange
+
+IPFS uses crypto:
+- cryptographic-hash content addressing
+- block-level deduplication
+- file integrity + versioning
+- filesystem-level encryption + signing support
+
+IPFS is p2p:
+- worldwide peer-to-peer file transfers
+- completely decentralized architecture
+- **no** central point of failure
+
+IPFS is a cdn:
+- add a file to the filesystem locally, and it's now available to the world
+- caching-friendly (content-hash naming)
+- bittorrent-based bandwidth distribution
+
+IPFS has a name service:
+- IPNS, an SFS inspired name system
+- global namespace based on PKI
+- serves to build trust chains
+- compatible with other NSes
+- can map DNS, .onion, .bit, etc to IPNS
+�

test/fixtures/v7-repo/blocks/EX/CIQKA7ZA5YM6KOJE3HVPZNQ6C4TJVSVGFTWWOTHP7GWZYGDUP5HIEXY.data

@@ -0,0 +1,4 @@
+/
+" �I%�s�!@��<��'����8���@:�шo_�direct�V2
+" �e��I�\(&PD�
+� �� 2
�hO.߃o� recursive�V

test/fixtures/v7-repo/blocks/IL/CIQJFGRQHQ45VCQLM7AJNF2GF5UHUAGGHC6LLAH6VYDEKLQMD4QLILY.data

@@ -0,0 +1,8 @@
+
+�
�
Come hang out in our IRC chat room if you have any questions.
+
+Contact the ipfs dev team:
+- Bugs: https://github.com/ipfs/go-ipfs/issues
+- Help: irc.freenode.org/#ipfs
+- Email: dev@ipfs.io
+�

test/fixtures/v7-repo/blocks/LG/CIQJBQD2O6K4CGJVCCTJNUP57QHR4SKHZ74OIITBBGLOMCO3ZOLWLGA.data

@@ -0,0 +1,9 @@
+
+��Some helpful resources for finding your way around ipfs:
+
+- quick-start: a quick show of various ipfs features.
+- ipfs commands: a list of all commands
+- ipfs --help: every command describes itself
+- https://github.com/ipfs/go-ipfs -- the src repository
+- #ipfs on irc.freenode.org -- the community irc channel
+�