From 78f69d7d7c34bce422091cf3bdde81cbab8892d9 Mon Sep 17 00:00:00 2001 From: Adam Uhlir Date: Sat, 11 May 2019 10:59:08 -0700 Subject: [PATCH 1/6] Integration of ipfs-repo-migrations The integration respect config's setting repoDisableAutoMigration that defines if migrations should be automatically applied or not. License: MIT Signed-off-by: Adam Uhlir --- package-list.json | 1 + package.json | 1 + src/core/boot.js | 25 +++++++++++++++++++++++++ src/core/components/repo.js | 21 +++++++++++++++++++++ src/core/config.js | 1 + 5 files changed, 49 insertions(+) diff --git a/package-list.json b/package-list.json index f51d47fa52..f959ca9c5e 100644 --- a/package-list.json +++ b/package-list.json @@ -18,6 +18,7 @@ "Repo", ["ipfs/js-ipfs-repo", "ipfs-repo"], + ["AuHau/js-ipfs-repo-migrations", "ipfs-repo-migrations"], "Exchange", ["ipfs/js-ipfs-block-service", "ipfs-block-service"], diff --git a/package.json b/package.json index ad8413336d..5a7606dc94 100644 --- a/package.json +++ b/package.json @@ -98,6 +98,7 @@ "ipfs-mfs": "~0.11.4", "ipfs-multipart": "~0.1.0", "ipfs-repo": "~0.26.6", + "ipfs-repo-migrations": "AuHau/js-ipfs-repo-migrations#dev", "ipfs-unixfs": "~0.1.16", "ipfs-unixfs-exporter": "~0.37.6", "ipfs-unixfs-importer": "~0.39.9", diff --git a/src/core/boot.js b/src/core/boot.js index 23e1992d61..32554221a9 100644 --- a/src/core/boot.js +++ b/src/core/boot.js @@ -27,6 +27,31 @@ module.exports = (self) => { cb(null, true) }) }, + // If migrations are enabled check & migrate repo + (repoOpened, cb) => { + // There is non-initialized repo => no migrations needed + if(!repoOpened){ + return cb(null, repoOpened) + } + + self.config.get('repoDisableAutoMigration') + .catch(err => { + if (!err.message.match('does not exist')) { + return Promise.reject(err) + } + + // Option not found, but default value is false, lets continue + return false + }) + .then(repoDisableAutoMigration => { + if (!repoDisableAutoMigration) { + self.log('checking for migrations') + return self.repo.migrate() + } + }) + .then(() => cb(null, repoOpened)) + .catch(cb) + }, (repoOpened, cb) => { // Init with existing initialized, opened, repo if (repoOpened) { diff --git a/src/core/components/repo.js b/src/core/components/repo.js index 23116d8cf5..bdb29dc457 100644 --- a/src/core/components/repo.js +++ b/src/core/components/repo.js @@ -2,6 +2,8 @@ const promisify = require('promisify-es6') const repoVersion = require('ipfs-repo').repoVersion +const log = require('debug')('ipfs:repo') +const migrator = require('ipfs-repo-migrations') module.exports = function repo (self) { return { @@ -38,6 +40,25 @@ module.exports = function repo (self) { }) }), + migrate: async function tryMigrateRepo () { + // Reads the repo version from datastore, not from the ipfs-repo package + const currentRepoVersion = await migrator.getCurrentRepoVersion(self._repo.path) + + if (currentRepoVersion >= repoVersion) { + if (currentRepoVersion > repoVersion) { + log('Your repo\'s version is higher then this version of js-ipfs require! You should revert it.') + } + + return // Nothing to migrate + } + + if (repoVersion > migrator.getLatestMigrationVersion()) { + throw new Error('The ipfs-repo-migrations package does not have migration for version: ' + repoVersion) + } + + return migrator.migrate(self._repo.path, repoVersion, true, self._repo.options) + }, + gc: promisify((options, callback) => { if (typeof options === 'function') { callback = options diff --git a/src/core/config.js b/src/core/config.js index 2fb66bb558..ba8f4bdf90 100644 --- a/src/core/config.js +++ b/src/core/config.js @@ -27,6 +27,7 @@ const s = superstruct({ const configSchema = s({ repo: optional(s('object|string')), repoOwner: 'boolean?', + repoDisableAutoMigration: 'boolean?', preload: s({ enabled: 'boolean?', addresses: optional(s(['multiaddr'])), From 9c4f7fc2ce1453164844628187014a257a562dc2 Mon Sep 17 00:00:00 2001 From: Adam Uhlir Date: Tue, 21 May 2019 11:43:44 -0700 Subject: [PATCH 2/6] Test coverage for migrations logic License: MIT Signed-off-by: Adam Uhlir --- test/core/migrations.spec.js | 172 +++++++++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 test/core/migrations.spec.js diff --git a/test/core/migrations.spec.js b/test/core/migrations.spec.js new file mode 100644 index 0000000000..0e92dc8c5b --- /dev/null +++ b/test/core/migrations.spec.js @@ -0,0 +1,172 @@ +/* eslint max-nested-callbacks: ["error", 8] */ +/* eslint-env mocha */ +'use strict' + +const chai = require('chai') +const dirtyChai = require('dirty-chai') +const expect = chai.expect +chai.use(dirtyChai) +const hat = require('hat') +const sinon = require('sinon') + +const IPFS = require('../../src/core') +const migrator = require('ipfs-repo-migrations') +const repoVersion = require('ipfs-repo').repoVersion + +// This gets replaced by `create-repo-browser.js` in the browser +const createTempRepo = require('../utils/create-repo-nodejs.js') + +async function initRepo (repo) { + const ipfs = new IPFS({ + repo: repo, + init: false, + start: false + }) + + await ipfs.init({ bits: 512, pass: hat() }) +} + +function setConfig (repo, key, value) { + return new Promise(((resolve, reject) => { + repo.config.set(key, value, (e) => { + if (e) { + return reject(e) + } + + resolve() + }) + } + )) +} + +describe('migrations', () => { + + let repo + let migrateStub + let currentVersionStub + let getLatestMigrationVersionStub + + before(() => { + currentVersionStub = sinon.stub(migrator, 'getCurrentRepoVersion') + migrateStub = sinon.stub(migrator, 'migrate') + getLatestMigrationVersionStub = sinon.stub(migrator, 'getLatestMigrationVersion') + }) + + after(() => { + currentVersionStub.restore() + migrateStub.restore() + }) + + beforeEach(() => { + sinon.reset() + repo = createTempRepo() + }) + + afterEach((done) => repo.teardown(done)) + + it('should migrate by default', (done) => { + migrateStub.resolves() + currentVersionStub.resolves(repoVersion - 1) + getLatestMigrationVersionStub.returns(repoVersion) + + initRepo(repo) + .then(() => { + + // Should not migrate when repo does not exists/is not initialized + expect(migrateStub.called).to.be.false() + + const migratingIpfs = new IPFS({ + repo: repo, + init: false, + start: false + }) + + migratingIpfs.on('ready', () => { + expect(migrateStub.called).to.be.true() + done() + }) + }) + }) + + it('should not migrate when repoDisableAutoMigration is true', (done) => { + migrateStub.resolves() + currentVersionStub.resolves(repoVersion - 1) + getLatestMigrationVersionStub.returns(repoVersion) + + initRepo(repo) + .then(() => setConfig(repo, 'repoDisableAutoMigration', true)) + .then(() => { + const migratingIpfs = new IPFS({ + repo: repo, + init: false, + start: false + }) + + migratingIpfs.on('ready', () => { + expect(migrateStub.called).to.be.false() + done() + }) + }) + }) + + it('should not migrate when versions matches', (done) => { + migrateStub.resolves() + currentVersionStub.resolves(repoVersion) + getLatestMigrationVersionStub.returns(repoVersion) + + initRepo(repo) + .then(() => { + const migratingIpfs = new IPFS({ + repo: repo, + init: false, + start: false + }) + + migratingIpfs.on('ready', () => { + expect(migrateStub.called).to.be.false() + done() + }) + }) + }) + + it('should not migrate when current repo versions is higher then expected', (done) => { + migrateStub.resolves() + currentVersionStub.resolves(repoVersion + 1) + getLatestMigrationVersionStub.returns(repoVersion) + + initRepo(repo) + .then(() => { + const migratingIpfs = new IPFS({ + repo: repo, + init: false, + start: false + }) + + migratingIpfs.on('ready', () => { + expect(migrateStub.called).to.be.false() + done() + }) + }) + }) + + it('should fail if ipfs-repo-migrations does not contain expected migration', (done) => { + getLatestMigrationVersionStub.returns(repoVersion - 1) + migrateStub.resolves() + currentVersionStub.resolves(repoVersion - 1) + + initRepo(repo) + .then(() => { + const migratingIpfs = new IPFS({ + repo: repo, + init: false, + start: false + }) + + migratingIpfs.on('error', (err) => { + expect(err).to.exist() + expect(err.message).to.match(/The ipfs-repo-migrations package does not have migration for version: /) + done() + }) + }) + }) +}) From cd78d408d6629f55e5e143e4a571d6b39be81fac Mon Sep 17 00:00:00 2001 From: Adam Uhlir Date: Tue, 21 May 2019 19:00:57 -0700 Subject: [PATCH 3/6] Description of migrations License: MIT Signed-off-by: Adam Uhlir --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 82c538d27b..5d35022ab8 100644 --- a/README.md +++ b/README.md @@ -199,6 +199,11 @@ node.on('ready', () => { }) ``` +### Migrations + +`js-ipfs` comes bundled with tool that automatically migrate the version of your IPFS repository when new version is available. +You can control this mechanism using config option `repoDisableAutoMigration` (`true`/`false` default is `false`). + ### [Tutorials and Examples](/examples) You can find some examples and tutorials in the [examples](/examples) folder, these exist to help you get started using `js-ipfs`. From 1f6d1e5fe7253b36bd92380f8603f909f2d95ae2 Mon Sep 17 00:00:00 2001 From: Adam Uhlir Date: Thu, 27 Jun 2019 12:03:29 +0200 Subject: [PATCH 4/6] Update migrate signature --- src/core/components/repo.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/core/components/repo.js b/src/core/components/repo.js index bdb29dc457..e75a498a23 100644 --- a/src/core/components/repo.js +++ b/src/core/components/repo.js @@ -49,14 +49,15 @@ module.exports = function repo (self) { log('Your repo\'s version is higher then this version of js-ipfs require! You should revert it.') } - return // Nothing to migrate + log('Nothing to migrate') + return } if (repoVersion > migrator.getLatestMigrationVersion()) { throw new Error('The ipfs-repo-migrations package does not have migration for version: ' + repoVersion) } - return migrator.migrate(self._repo.path, repoVersion, true, self._repo.options) + return migrator.migrate(self._repo.path, {toVersion: repoVersion, ignoreLock: true, repoOptions: self._repo.options}) }, gc: promisify((options, callback) => { From 9324aa4b2a6bc3f458f61b083e12e765478e2c83 Mon Sep 17 00:00:00 2001 From: Adam Uhlir Date: Fri, 28 Jun 2019 12:43:23 +0200 Subject: [PATCH 5/6] Revert "Integration of ipfs-repo-migrations" This reverts commit 78f69d7d7c34bce422091cf3bdde81cbab8892d9. --- package-list.json | 1 - package.json | 1 - src/core/boot.js | 25 ------------------------- src/core/components/repo.js | 22 ---------------------- src/core/config.js | 1 - 5 files changed, 50 deletions(-) diff --git a/package-list.json b/package-list.json index f959ca9c5e..f51d47fa52 100644 --- a/package-list.json +++ b/package-list.json @@ -18,7 +18,6 @@ "Repo", ["ipfs/js-ipfs-repo", "ipfs-repo"], - ["AuHau/js-ipfs-repo-migrations", "ipfs-repo-migrations"], "Exchange", ["ipfs/js-ipfs-block-service", "ipfs-block-service"], diff --git a/package.json b/package.json index 5a7606dc94..ad8413336d 100644 --- a/package.json +++ b/package.json @@ -98,7 +98,6 @@ "ipfs-mfs": "~0.11.4", "ipfs-multipart": "~0.1.0", "ipfs-repo": "~0.26.6", - "ipfs-repo-migrations": "AuHau/js-ipfs-repo-migrations#dev", "ipfs-unixfs": "~0.1.16", "ipfs-unixfs-exporter": "~0.37.6", "ipfs-unixfs-importer": "~0.39.9", diff --git a/src/core/boot.js b/src/core/boot.js index 32554221a9..23e1992d61 100644 --- a/src/core/boot.js +++ b/src/core/boot.js @@ -27,31 +27,6 @@ module.exports = (self) => { cb(null, true) }) }, - // If migrations are enabled check & migrate repo - (repoOpened, cb) => { - // There is non-initialized repo => no migrations needed - if(!repoOpened){ - return cb(null, repoOpened) - } - - self.config.get('repoDisableAutoMigration') - .catch(err => { - if (!err.message.match('does not exist')) { - return Promise.reject(err) - } - - // Option not found, but default value is false, lets continue - return false - }) - .then(repoDisableAutoMigration => { - if (!repoDisableAutoMigration) { - self.log('checking for migrations') - return self.repo.migrate() - } - }) - .then(() => cb(null, repoOpened)) - .catch(cb) - }, (repoOpened, cb) => { // Init with existing initialized, opened, repo if (repoOpened) { diff --git a/src/core/components/repo.js b/src/core/components/repo.js index e75a498a23..23116d8cf5 100644 --- a/src/core/components/repo.js +++ b/src/core/components/repo.js @@ -2,8 +2,6 @@ const promisify = require('promisify-es6') const repoVersion = require('ipfs-repo').repoVersion -const log = require('debug')('ipfs:repo') -const migrator = require('ipfs-repo-migrations') module.exports = function repo (self) { return { @@ -40,26 +38,6 @@ module.exports = function repo (self) { }) }), - migrate: async function tryMigrateRepo () { - // Reads the repo version from datastore, not from the ipfs-repo package - const currentRepoVersion = await migrator.getCurrentRepoVersion(self._repo.path) - - if (currentRepoVersion >= repoVersion) { - if (currentRepoVersion > repoVersion) { - log('Your repo\'s version is higher then this version of js-ipfs require! You should revert it.') - } - - log('Nothing to migrate') - return - } - - if (repoVersion > migrator.getLatestMigrationVersion()) { - throw new Error('The ipfs-repo-migrations package does not have migration for version: ' + repoVersion) - } - - return migrator.migrate(self._repo.path, {toVersion: repoVersion, ignoreLock: true, repoOptions: self._repo.options}) - }, - gc: promisify((options, callback) => { if (typeof options === 'function') { callback = options diff --git a/src/core/config.js b/src/core/config.js index ba8f4bdf90..2fb66bb558 100644 --- a/src/core/config.js +++ b/src/core/config.js @@ -27,7 +27,6 @@ const s = superstruct({ const configSchema = s({ repo: optional(s('object|string')), repoOwner: 'boolean?', - repoDisableAutoMigration: 'boolean?', preload: s({ enabled: 'boolean?', addresses: optional(s(['multiaddr'])), From 4654e9eb06081a091d56dc875c2a738f5fbee7cb Mon Sep 17 00:00:00 2001 From: Adam Uhlir Date: Fri, 28 Jun 2019 12:43:38 +0200 Subject: [PATCH 6/6] Revert "Test coverage for migrations logic" This reverts commit 9c4f7fc2ce1453164844628187014a257a562dc2. --- test/core/migrations.spec.js | 172 ----------------------------------- 1 file changed, 172 deletions(-) delete mode 100644 test/core/migrations.spec.js diff --git a/test/core/migrations.spec.js b/test/core/migrations.spec.js deleted file mode 100644 index 0e92dc8c5b..0000000000 --- a/test/core/migrations.spec.js +++ /dev/null @@ -1,172 +0,0 @@ -/* eslint max-nested-callbacks: ["error", 8] */ -/* eslint-env mocha */ -'use strict' - -const chai = require('chai') -const dirtyChai = require('dirty-chai') -const expect = chai.expect -chai.use(dirtyChai) -const hat = require('hat') -const sinon = require('sinon') - -const IPFS = require('../../src/core') -const migrator = require('ipfs-repo-migrations') -const repoVersion = require('ipfs-repo').repoVersion - -// This gets replaced by `create-repo-browser.js` in the browser -const createTempRepo = require('../utils/create-repo-nodejs.js') - -async function initRepo (repo) { - const ipfs = new IPFS({ - repo: repo, - init: false, - start: false - }) - - await ipfs.init({ bits: 512, pass: hat() }) -} - -function setConfig (repo, key, value) { - return new Promise(((resolve, reject) => { - repo.config.set(key, value, (e) => { - if (e) { - return reject(e) - } - - resolve() - }) - } - )) -} - -describe('migrations', () => { - - let repo - let migrateStub - let currentVersionStub - let getLatestMigrationVersionStub - - before(() => { - currentVersionStub = sinon.stub(migrator, 'getCurrentRepoVersion') - migrateStub = sinon.stub(migrator, 'migrate') - getLatestMigrationVersionStub = sinon.stub(migrator, 'getLatestMigrationVersion') - }) - - after(() => { - currentVersionStub.restore() - migrateStub.restore() - }) - - beforeEach(() => { - sinon.reset() - repo = createTempRepo() - }) - - afterEach((done) => repo.teardown(done)) - - it('should migrate by default', (done) => { - migrateStub.resolves() - currentVersionStub.resolves(repoVersion - 1) - getLatestMigrationVersionStub.returns(repoVersion) - - initRepo(repo) - .then(() => { - - // Should not migrate when repo does not exists/is not initialized - expect(migrateStub.called).to.be.false() - - const migratingIpfs = new IPFS({ - repo: repo, - init: false, - start: false - }) - - migratingIpfs.on('ready', () => { - expect(migrateStub.called).to.be.true() - done() - }) - }) - }) - - it('should not migrate when repoDisableAutoMigration is true', (done) => { - migrateStub.resolves() - currentVersionStub.resolves(repoVersion - 1) - getLatestMigrationVersionStub.returns(repoVersion) - - initRepo(repo) - .then(() => setConfig(repo, 'repoDisableAutoMigration', true)) - .then(() => { - const migratingIpfs = new IPFS({ - repo: repo, - init: false, - start: false - }) - - migratingIpfs.on('ready', () => { - expect(migrateStub.called).to.be.false() - done() - }) - }) - }) - - it('should not migrate when versions matches', (done) => { - migrateStub.resolves() - currentVersionStub.resolves(repoVersion) - getLatestMigrationVersionStub.returns(repoVersion) - - initRepo(repo) - .then(() => { - const migratingIpfs = new IPFS({ - repo: repo, - init: false, - start: false - }) - - migratingIpfs.on('ready', () => { - expect(migrateStub.called).to.be.false() - done() - }) - }) - }) - - it('should not migrate when current repo versions is higher then expected', (done) => { - migrateStub.resolves() - currentVersionStub.resolves(repoVersion + 1) - getLatestMigrationVersionStub.returns(repoVersion) - - initRepo(repo) - .then(() => { - const migratingIpfs = new IPFS({ - repo: repo, - init: false, - start: false - }) - - migratingIpfs.on('ready', () => { - expect(migrateStub.called).to.be.false() - done() - }) - }) - }) - - it('should fail if ipfs-repo-migrations does not contain expected migration', (done) => { - getLatestMigrationVersionStub.returns(repoVersion - 1) - migrateStub.resolves() - currentVersionStub.resolves(repoVersion - 1) - - initRepo(repo) - .then(() => { - const migratingIpfs = new IPFS({ - repo: repo, - init: false, - start: false - }) - - migratingIpfs.on('error', (err) => { - expect(err).to.exist() - expect(err.message).to.match(/The ipfs-repo-migrations package does not have migration for version: /) - done() - }) - }) - }) -})