From 2148c6bbf3a2bea3c4b57954ea73c2ce9f42e4d0 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 6 Aug 2020 12:31:24 +0100 Subject: [PATCH 1/4] fix: require passing repo options to migrator In order to successfully open a datastore we should accept config that tells us how to open the datastore. Refs: https://github.com/ipfs-shipyard/ipld-explorer/pull/65 --- README.md | 24 +++++++++---- package.json | 1 + src/commands.js | 4 +-- src/errors.js | 15 ++++++++ src/index.js | 10 +++--- src/repo/init.js | 27 ++++++++------ src/repo/version.js | 45 +++++++++++++++-------- src/utils.js | 4 +++ test/browser.js | 69 +++++++++++++++++++++++++++++------ test/init-test.js | 56 +++++++++++++++++++---------- test/integration-test.js | 39 ++++++++++++++------ test/node.js | 77 +++++++++++++++++++++++++++++++--------- test/version-test.js | 66 ++++++++++++++++++++-------------- 13 files changed, 315 insertions(+), 122 deletions(-) diff --git a/README.md b/README.md index eaf75e2..f82df1e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Migration tool for JS IPFS Repo +# Migration tool for JS IPFS Repo [![Travis CI](https://flat.badgen.net/travis/ipfs/js-ipfs-repo-migrations)](https://travis-ci.com/ipfs/js-ipfs-repo-migrations) [![codecov](https://codecov.io/gh/ipfs/js-ipfs-repo-migrations/branch/master/graph/badge.svg)](https://codecov.io/gh/ipfs/js-ipfs-repo-migrations) @@ -15,36 +15,41 @@ This package is inspired by the [go-ipfs repo migration tool](https://github.com/ipfs/fs-repo-migrations/) -## Lead Maintainer +## Lead Maintainer [Adam Uhlíř](https://github.com/auhau/) -## Table of Contents +## Table of Contents - [Background](#background) - [Install](#install) - [npm](#npm) - [Use in Node.js](#use-in-nodejs) - [Use in a browser with browserify, webpack or any other bundler](#use-in-a-browser-with-browserify-webpack-or-any-other-bundler) - - [Use in a browser Using a script tag](#use-in-a-browser-using-a-script-tag) - [Usage](#usage) - [API](#api) + - [`.migrate(path, toVersion, {ignoreLock, repoOptions, onProgress, isDryRun}) -> Promise`](#migratepath-toversion-ignorelock-repooptions-onprogress-isdryrun---promisevoid) + - [`onProgress(migration, counter, totalMigrations)`](#onprogressmigration-counter-totalmigrations) + - [`.revert(path, toVersion, {ignoreLock, repoOptions, onProgress, isDryRun}) -> Promise`](#revertpath-toversion-ignorelock-repooptions-onprogress-isdryrun---promisevoid) + - [`getLatestMigrationVersion() -> int`](#getlatestmigrationversion---int) - [CLI](#cli) - [Creating a new migration](#creating-a-new-migration) - [Architecture of a migration](#architecture-of-a-migration) + - [`.migrate(repoPath, repoOptions)`](#migraterepopath-repooptions) + - [`.revert(repoPath, repoOptions)`](#revertrepopath-repooptions) - [Browser vs. NodeJS environments](#browser-vs-nodejs-environments) - [Guidelines](#guidelines) - [Integration with js-ipfs](#integration-with-js-ipfs) + - [Tests](#tests) - [Empty migrations](#empty-migrations) - [Migrations matrix](#migrations-matrix) - [Developer](#developer) - - [Module versioning notes](#module-versioning-notes) + - [Module versioning notes](#module-versioning-notes) - [Contribute](#contribute) - [License](#license) ## Background - As js-ipfs evolves and new technologies, algorithms and data structures are incorporated it is necessary to enable users to transition between versions. Different versions of js-ipfs may expect a different IPFS repo structure or content (see: [IPFS repo spec](https://github.com/ipfs/specs/tree/master/repo), [JS implementation](https://github.com/ipfs/js-ipfs-repo) ). So the IPFS repo is versioned, and this package provides a framework to create migrations to transition @@ -87,10 +92,15 @@ const migrations = require('ipfs-repo-migrations') const repoPath = 'some/repo/path' const currentRepoVersion = 7 const latestVersion = migrations.getLatestMigrationVersion() +const repoOptions = { + ... // the same storage backend/storage options passed to `ipfs-repo` +} if(currentRepoVersion < latestVersion){ // Old repo! Lets migrate to latest version! - await migrations.migrate(repoPath, latestVersion) + await migrations.migrate(repoPath, latestVersion, { + repoOptions + }) } ``` diff --git a/package.json b/package.json index c342db8..8a019da 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "main": "src/index.js", "browser": { "./src/repo/lock.js": "./src/repo/lock-memory.js", + "./src/repo/default-root-options.js": "./src/repo/ldefault-root-options.browser.js", "datastore-fs": "datastore-level" }, "bin": { diff --git a/src/commands.js b/src/commands.js index bcb2450..04a23e9 100644 --- a/src/commands.js +++ b/src/commands.js @@ -43,7 +43,7 @@ function reportingClosure (action) { process.stdout.write(`${chalk.green(`[${currentlyMigrated}/${totalToMigrate}]`)} Successfully ${action} ${chalk.bold(migration.version)}: ${migration.description}\n`) } -async function migrate ({ repoPath, migrations, to, dry, revertOk }) { +async function migrate ({ repoPath, repoOptions, migrations, to, dry, revertOk }) { repoPath = repoPath || process.env.IPFS_PATH || path.join(os.homedir(), '.jsipfs') migrations = migrations === undefined ? require('../migrations') : require(migrations) @@ -51,7 +51,7 @@ async function migrate ({ repoPath, migrations, to, dry, revertOk }) { to = migrator.getLatestMigrationVersion(migrations) } - const version = await repoVersion.getVersion(repoPath) + const version = await repoVersion.getVersion(repoPath, repoOptions) let action if (dry) { diff --git a/src/errors.js b/src/errors.js index 1aaa2f1..592f783 100644 --- a/src/errors.js +++ b/src/errors.js @@ -60,3 +60,18 @@ class InvalidValueError extends Error { InvalidValueError.code = 'ERR_INVALID_VALUE' exports.InvalidValueError = InvalidValueError + +/** + * Exception raised when config is not passed. + */ +class MissingRepoOptionsError extends Error { + constructor (message) { + super(message) + this.name = 'MissingRepoOptionsError' + this.code = 'ERR_MISSING_REPO_OPTIONS' + this.message = message + } +} + +MissingRepoOptionsError.code = 'ERR_MISSING_REPO_OPTIONS' +exports.MissingRepoOptionsError = MissingRepoOptionsError diff --git a/src/index.js b/src/index.js index bdb7051..5a268a6 100644 --- a/src/index.js +++ b/src/index.js @@ -61,7 +61,7 @@ async function migrate (path, toVersion, { ignoreLock = false, repoOptions, onPr throw new errors.InvalidValueError('Version has to be positive integer!') } - const currentVersion = await repoVersion.getVersion(path) + const currentVersion = await repoVersion.getVersion(path, repoOptions) if (currentVersion === toVersion) { log('Nothing to migrate.') @@ -105,7 +105,7 @@ async function migrate (path, toVersion, { ignoreLock = false, repoOptions, onPr log(`Migrating to version ${migration.version} finished`) } - if (!isDryRun) await repoVersion.setVersion(path, toVersion || getLatestMigrationVersion(migrations)) + if (!isDryRun) await repoVersion.setVersion(path, toVersion || getLatestMigrationVersion(migrations), repoOptions) log('Repo successfully migrated ', toVersion !== undefined ? `to version ${toVersion}!` : 'to latest version!') } finally { if (!isDryRun && !ignoreLock) await lock.close() @@ -146,7 +146,7 @@ async function revert (path, toVersion, { ignoreLock = false, repoOptions, onPro throw new errors.InvalidValueError('Version has to be positive integer!') } - const currentVersion = await repoVersion.getVersion(path) + const currentVersion = await repoVersion.getVersion(path, repoOptions) if (currentVersion === toVersion) { log('Nothing to revert.') return @@ -182,7 +182,7 @@ async function revert (path, toVersion, { ignoreLock = false, repoOptions, onPro } catch (e) { const lastSuccessfullyRevertedVersion = migration.version log(`An exception was raised during execution of migration. Setting the repo's version to last successfully reverted version: ${lastSuccessfullyRevertedVersion}`) - await repoVersion.setVersion(path, lastSuccessfullyRevertedVersion) + await repoVersion.setVersion(path, lastSuccessfullyRevertedVersion, repoOptions) e.message = `During reversion to version ${migration.version} exception was raised: ${e.message}` throw e @@ -192,7 +192,7 @@ async function revert (path, toVersion, { ignoreLock = false, repoOptions, onPro log(`Reverting to version ${migration.version} finished`) } - if (!isDryRun) await repoVersion.setVersion(path, toVersion) + if (!isDryRun) await repoVersion.setVersion(path, toVersion, repoOptions) log(`All migrations successfully reverted to version ${toVersion}!`) } finally { if (!isDryRun && !ignoreLock) await lock.close() diff --git a/src/repo/init.js b/src/repo/init.js index bf5b8cb..400dbb0 100644 --- a/src/repo/init.js +++ b/src/repo/init.js @@ -1,20 +1,25 @@ 'use strict' -const Datastore = require('datastore-fs') const log = require('debug')('repo-migrations:repo:init') +const { CONFIG_KEY, VERSION_KEY, getDatastoreAndOptions } = require('../utils') +const { MissingRepoOptionsError } = require('../errors') -const Key = require('interface-datastore').Key - -const versionKey = new Key('/version') -const configKey = new Key('/config') +exports.isRepoInitialized = async function isRepoInitialized (path, repoOptions) { + if (!repoOptions) { + throw new MissingRepoOptionsError('Please pass repo options when trying to open a repo') + } -exports.isRepoInitialized = async function isRepoInitialized (path) { let root try { - root = new Datastore(path, { extension: '', createIfMissing: false }) + const { + StorageBackend, + storageOptions + } = getDatastoreAndOptions(repoOptions, 'root') + + root = new StorageBackend(path, storageOptions) await root.open() - const versionCheck = await root.has(versionKey) - const configCheck = await root.has(configKey) + const versionCheck = await root.has(VERSION_KEY) + const configCheck = await root.has(CONFIG_KEY) if (!versionCheck || !configCheck) { log(`Version entry present: ${versionCheck}`) log(`Config entry present: ${configCheck}`) @@ -26,6 +31,8 @@ exports.isRepoInitialized = async function isRepoInitialized (path) { log('While checking if repo is initialized error was thrown: ' + e.message) return false } finally { - if (root !== undefined) await root.close() + if (root !== undefined) { + await root.close() + } } } diff --git a/src/repo/version.js b/src/repo/version.js index 34ec0ff..51a8e6c 100644 --- a/src/repo/version.js +++ b/src/repo/version.js @@ -1,13 +1,9 @@ 'use strict' const { Buffer } = require('buffer') -const errors = require('../errors') const repoInit = require('./init') -const Datastore = require('datastore-fs') - -const Key = require('interface-datastore').Key - -const versionKey = new Key('version') +const { MissingRepoOptionsError, NotInitializedRepoError } = require('../errors') +const { VERSION_KEY, getDatastoreAndOptions } = require('../utils') exports.getVersion = getVersion @@ -17,17 +13,27 @@ exports.getVersion = getVersion * even in case of change of repo's versioning. * * @param {string} path + * @param {Object} repoOptions Options used to create a repo, the same as pased to ipfs-repo * @returns {Promise} */ -async function getVersion (path) { - if (!(await repoInit.isRepoInitialized(path))) { - throw new errors.NotInitializedRepoError(`Repo in path ${path} is not initialized!`) +async function getVersion (path, repoOptions) { + if (!(await repoInit.isRepoInitialized(path, repoOptions))) { + throw new NotInitializedRepoError(`Repo in path ${path} is not initialized!`) } - const store = new Datastore(path, { extension: '', createIfMissing: false }) + if (!repoOptions) { + throw new MissingRepoOptionsError('Please pass repo options when trying to open a repo') + } + + const { + StorageBackend, + storageOptions + } = getDatastoreAndOptions(repoOptions, 'root') + + const store = new StorageBackend(path, storageOptions) await store.open() - const version = parseInt(await store.get(versionKey)) + const version = parseInt(await store.get(VERSION_KEY)) await store.close() return version @@ -38,12 +44,23 @@ async function getVersion (path) { * * @param {string} path * @param {int} version + * @param {Object} repoOptions Options used to create a repo, the same as pased to ipfs-repo * @returns {Promise} */ -async function setVersion (path, version) { - const store = new Datastore(path, { extension: '', createIfMissing: false }) +async function setVersion (path, version, repoOptions) { + if (!repoOptions) { + throw new MissingRepoOptionsError('Please pass repo options when trying to open a repo') + } + + const { + StorageBackend, + storageOptions + } = getDatastoreAndOptions(repoOptions, 'root') + + const store = new StorageBackend(path, storageOptions) await store.open() - await store.put(versionKey, Buffer.from(String(version))) + + await store.put(VERSION_KEY, Buffer.from(String(version))) await store.close() } diff --git a/src/utils.js b/src/utils.js index b180af1..62e279d 100644 --- a/src/utils.js +++ b/src/utils.js @@ -1,6 +1,10 @@ 'use strict' const Datastore = require('datastore-fs') +const Key = require('interface-datastore').Key + +exports.CONFIG_KEY = new Key('/config') +exports.VERSION_KEY = new Key('/version') exports.getDatastoreAndOptions = function getDatastoreAndOptions (options, key, defaultDatastore = Datastore) { let StorageBackend, storageBackendOptions diff --git a/test/browser.js b/test/browser.js index a8f55dd..58d7da0 100644 --- a/test/browser.js +++ b/test/browser.js @@ -3,30 +3,77 @@ const { Buffer } = require('buffer') const loadFixture = require('aegir/fixtures') -const Datastore = require('datastore-level') +const { CONFIG_KEY, VERSION_KEY, getDatastoreAndOptions } = require('../src/utils') -const Key = require('interface-datastore').Key -const CONFIG_KEY = new Key('config') -const VERSION_KEY = new Key('version') +const repoOptions = { + lock: 'memory', + storageBackends: { + root: require('datastore-level'), + blocks: require('datastore-level'), + keys: require('datastore-level'), + datastore: require('datastore-level'), + pins: require('datastore-level') + }, + storageBackendOptions: { + root: { + extension: '', + prefix: '', + version: 2 + }, + blocks: { + sharding: false, + prefix: '', + version: 2 + }, + keys: { + sharding: false, + prefix: '', + version: 2 + }, + datastore: { + sharding: false, + prefix: '', + version: 2 + } + } +} async function createRepo () { + const { + StorageBackend, + storageOptions + } = getDatastoreAndOptions(repoOptions, 'root') + const date = Date.now().toString() const dir = 'test-repo-for-' + date - const store = new Datastore(dir, { extension: '', createIfMissing: true }) + const store = new StorageBackend(dir, { + ...storageOptions, + createIfMissing: true + }) await store.open() await store.close() return dir } async function createAndLoadRepo () { + const { + StorageBackend, + storageOptions + } = getDatastoreAndOptions(repoOptions, 'root') + const date = Date.now().toString() const dir = 'test-repo-for-' + date - const store = new Datastore(dir, { extension: '', createIfMissing: true }) + const store = new StorageBackend(dir, { + ...storageOptions, + createIfMissing: true + }) await store.open() await store.put(VERSION_KEY, Buffer.from(loadFixture('test/fixtures/test-repo/version'))) await store.put(CONFIG_KEY, Buffer.from(loadFixture('test/fixtures/test-repo/config'))) + await store.close() + return dir } @@ -36,23 +83,23 @@ async function repoCleanup (dir) { describe('Browser specific tests', () => { describe('lock.js tests', () => { describe('mem-lock tests', () => { - require('./lock-test')(require('../src/repo/lock-memory'), createRepo, repoCleanup) + require('./lock-test')(require('../src/repo/lock-memory'), createRepo, repoCleanup, repoOptions) }) }) describe('version tests', () => { - require('./version-test')(createRepo, repoCleanup) + require('./version-test')(createRepo, repoCleanup, repoOptions) }) describe('migrations tests', () => { - require('./migrations/migration-8-test')(createRepo, repoCleanup) + require('./migrations/migration-8-test')(createRepo, repoCleanup, repoOptions) }) describe('init tests', () => { - require('./init-test')(createRepo, repoCleanup) + require('./init-test')(createRepo, repoCleanup, repoOptions) }) describe('integration tests', () => { - require('./integration-test')(createAndLoadRepo, repoCleanup) + require('./integration-test')(createAndLoadRepo, repoCleanup, repoOptions) }) }) diff --git a/test/init-test.js b/test/init-test.js index 0ceb63d..4626f11 100644 --- a/test/init-test.js +++ b/test/init-test.js @@ -3,12 +3,10 @@ const { Buffer } = require('buffer') const { expect } = require('./util') - -const Datastore = require('datastore-fs') -const Key = require('interface-datastore').Key +const { CONFIG_KEY, VERSION_KEY, getDatastoreAndOptions } = require('../src/utils') const repoInit = require('../src/repo/init') -module.exports = (setup, cleanup) => { +module.exports = (setup, cleanup, repoOptions) => { let dir beforeEach(async () => { @@ -19,38 +17,58 @@ module.exports = (setup, cleanup) => { ) it('should return true with valid initialized repo', async () => { - const versionKey = new Key('version') - const configKey = new Key('config') - const store = new Datastore(dir, { extension: '', createIfMissing: false }) + const { + StorageBackend, + storageOptions + } = getDatastoreAndOptions(repoOptions, 'root') + + const store = new StorageBackend(dir, { + ...storageOptions, + createIfMissing: false + }) await store.open() - await store.put(versionKey, Buffer.from('7')) - await store.put(configKey, Buffer.from('config')) + await store.put(VERSION_KEY, Buffer.from('7')) + await store.put(CONFIG_KEY, Buffer.from('config')) await store.close() - expect(await repoInit.isRepoInitialized(dir)).to.be.true() + expect(await repoInit.isRepoInitialized(dir, repoOptions)).to.be.true() }) it('should return false with missing version key', async () => { - const configKey = new Key('config') - const store = new Datastore(dir, { extension: '', createIfMissing: false }) + const { + StorageBackend, + storageOptions + } = getDatastoreAndOptions(repoOptions, 'root') + + const store = new StorageBackend(dir, { + ...storageOptions, + createIfMissing: false + }) await store.open() - await store.put(configKey, '') + await store.put(CONFIG_KEY, '') await store.close() - expect(await repoInit.isRepoInitialized(dir)).to.be.false() + expect(await repoInit.isRepoInitialized(dir, repoOptions)).to.be.false() }) it('should return false with missing config key', async () => { - const versionKey = new Key('version') - const store = new Datastore(dir, { extension: '', createIfMissing: false }) + const { + StorageBackend, + storageOptions + } = getDatastoreAndOptions(repoOptions, 'root') + + const store = new StorageBackend(dir, { + ...storageOptions, + createIfMissing: false + }) await store.open() - await store.put(versionKey, '') + await store.put(VERSION_KEY, '') await store.close() - expect(await repoInit.isRepoInitialized(dir)).to.be.false() + expect(await repoInit.isRepoInitialized(dir, repoOptions)).to.be.false() }) it('should return false if the repo does not exists', async () => { - return expect(await repoInit.isRepoInitialized('/some/random/dirrr')).to.be.false() + return expect(await repoInit.isRepoInitialized('/some/random/dirrr', repoOptions)).to.be.false() }) } diff --git a/test/integration-test.js b/test/integration-test.js index e09b929..5f78134 100644 --- a/test/integration-test.js +++ b/test/integration-test.js @@ -5,13 +5,9 @@ const { expect } = require('./util') const migrator = require('../src') const migrations = require('./test-migrations') +const { VERSION_KEY, CONFIG_KEY, getDatastoreAndOptions } = require('../src/utils') -const Datastore = require('datastore-fs') -const Key = require('interface-datastore').Key -const CONFIG_KEY = new Key('config') -const VERSION_KEY = new Key('version') - -module.exports = (setup, cleanup) => { +module.exports = (setup, cleanup, repoOptions) => { let dir beforeEach(async () => { @@ -22,9 +18,18 @@ module.exports = (setup, cleanup) => { ) it('migrate forward', async () => { - await migrator.migrate(dir, migrator.getLatestMigrationVersion(migrations), { migrations: migrations }) + await migrator.migrate(dir, migrator.getLatestMigrationVersion(migrations), { + migrations: migrations, + repoOptions + }) + + const { + StorageBackend, + storageOptions + } = getDatastoreAndOptions(repoOptions, 'root') + + const store = new StorageBackend(dir, storageOptions) - const store = new Datastore(dir, { extension: '', createIfMissing: false }) await store.open() const version = await store.get(VERSION_KEY) expect(version.toString()).to.be.equal('2') @@ -36,11 +41,23 @@ module.exports = (setup, cleanup) => { }) it('revert', async () => { - await migrator.migrate(dir, migrator.getLatestMigrationVersion(migrations), { migrations: migrations }) + await migrator.migrate(dir, migrator.getLatestMigrationVersion(migrations), { + migrations: migrations, + repoOptions + }) + + await migrator.revert(dir, 1, { + migrations: migrations, + repoOptions + }) + + const { + StorageBackend, + storageOptions + } = getDatastoreAndOptions(repoOptions, 'root') - await migrator.revert(dir, 1, { migrations: migrations }) + const store = new StorageBackend(dir, storageOptions) - const store = new Datastore(dir, { extension: '', createIfMissing: false }) await store.open() const version = await store.get(VERSION_KEY) expect(version.toString()).to.be.equal('1') diff --git a/test/node.js b/test/node.js index 584275d..b02c2d6 100644 --- a/test/node.js +++ b/test/node.js @@ -3,24 +3,67 @@ const promisify = require('util').promisify const asyncRimraf = promisify(require('rimraf')) -const asyncNcp = promisify(require('ncp').ncp) -const path = require('path') -const fs = require('fs') +const loadFixture = require('aegir/fixtures') +const { CONFIG_KEY, VERSION_KEY, getDatastoreAndOptions } = require('../src/utils') -function createRepo () { - const testRepoPath = path.join(__dirname, 'fixtures', 'test-repo') - const date = Date.now().toString() - const dir = testRepoPath + '-for-' + date - fs.mkdirSync(dir) +const repoOptions = { + lock: 'fs', + storageBackends: { + root: require('datastore-fs'), + blocks: require('datastore-fs'), + keys: require('datastore-fs'), + datastore: require('datastore-level'), + pins: require('datastore-level') + }, + storageBackendOptions: { + root: { + extension: '' + }, + blocks: { + sharding: true, + extension: '.data' + }, + keys: { + } + } +} + +async function createRepo () { + const { + StorageBackend, + storageOptions + } = getDatastoreAndOptions(repoOptions, 'root') + const date = Date.now().toString() + const dir = 'test-repo-for-' + date + const store = new StorageBackend(dir, { + ...storageOptions, + createIfMissing: true + }) + await store.open() + await store.close() return dir } async function createAndLoadRepo () { - const dir = createRepo() - const testRepoPath = path.join(__dirname, 'fixtures', 'test-repo') + const { + StorageBackend, + storageOptions + } = getDatastoreAndOptions(repoOptions, 'root') + + const date = Date.now().toString() + const dir = 'test-repo-for-' + date + const store = new StorageBackend(dir, { + ...storageOptions, + createIfMissing: true + }) + await store.open() + + await store.put(VERSION_KEY, Buffer.from(loadFixture('test/fixtures/test-repo/version'))) + await store.put(CONFIG_KEY, Buffer.from(loadFixture('test/fixtures/test-repo/config'))) + + await store.close() - await asyncNcp(testRepoPath, dir) return dir } @@ -31,27 +74,27 @@ function repoCleanup (dir) { describe('Node specific tests', () => { describe('lock.js tests', () => { describe('fs-lock tests', () => { - require('./lock-test')(require('../src/repo/lock'), createRepo, repoCleanup) + require('./lock-test')(require('../src/repo/lock'), createRepo, repoCleanup, repoOptions) }) describe('mem-lock tests', () => { - require('./lock-test')(require('../src/repo/lock-memory'), createRepo, repoCleanup) + require('./lock-test')(require('../src/repo/lock-memory'), createRepo, repoCleanup, repoOptions) }) }) describe('version tests', () => { - require('./version-test')(createRepo, repoCleanup) + require('./version-test')(createRepo, repoCleanup, repoOptions) }) describe('migrations tests', () => { - require('./migrations/migration-8-test')(createRepo, repoCleanup) + require('./migrations/migration-8-test')(createRepo, repoCleanup, repoOptions) }) describe('init tests', () => { - require('./init-test')(createRepo, repoCleanup) + require('./init-test')(createRepo, repoCleanup, repoOptions) }) describe('integration tests', () => { - require('./integration-test')(createAndLoadRepo, repoCleanup) + require('./integration-test')(createAndLoadRepo, repoCleanup, repoOptions) }) }) diff --git a/test/version-test.js b/test/version-test.js index 20aac49..08355a6 100644 --- a/test/version-test.js +++ b/test/version-test.js @@ -3,9 +3,7 @@ const { Buffer } = require('buffer') const { expect } = require('./util') - -const Datastore = require('datastore-fs') -const Key = require('interface-datastore').Key +const { VERSION_KEY, CONFIG_KEY, getDatastoreAndOptions } = require('../src/utils') const version = require('../src/repo/version') const errors = require('../src/errors') @@ -13,46 +11,62 @@ const errors = require('../src/errors') // When new versioning mechanism is introduced in new version don't forget to update // the range (from/to) of the previous version test's description -module.exports = (setup, cleanup) => { - it('getVersion should fail without any version in repo', async () => { - const dir = await setup() - await expect(version.getVersion(dir)).to.be.eventually.rejectedWith(errors.NotInitializedRepoError).with.property('code', errors.NotInitializedRepoError.code) - return cleanup(dir) +module.exports = (setup, cleanup, repoOptions) => { + let dir + + beforeEach(async () => { + dir = await setup() }) - describe('version 7 and bellow', () => { - let dir + afterEach(() => cleanup(dir)) - beforeEach(async () => { - dir = await setup() - }) - afterEach(() => - cleanup(dir) - ) + it('getVersion should fail without any version in repo', async () => { + await expect(version.getVersion(dir, repoOptions)).to.be.eventually.rejectedWith(errors.NotInitializedRepoError) + .with.property('code', errors.NotInitializedRepoError.code) + }) + describe('version 7 and bellow', () => { it('should get version number', async () => { // Create version file - const store = new Datastore(dir, { extension: '', createIfMissing: false }) + const { + StorageBackend, + storageOptions + } = getDatastoreAndOptions(repoOptions, 'root') + + const store = new StorageBackend(dir, { + ...storageOptions, + createIfMissing: false + }) + await store.open() - await store.put(new Key('config'), Buffer.from('some dummy config')) - await store.put(new Key('version'), Buffer.from('7')) + await store.put(CONFIG_KEY, Buffer.from('some dummy config')) + await store.put(VERSION_KEY, Buffer.from('7')) await store.close() - expect(await version.getVersion(dir)).to.be.equal(7) + expect(await version.getVersion(dir, repoOptions)).to.be.equal(7) }) it('should set version number', async () => { - await expect(version.getVersion(dir)).to.be.eventually.rejectedWith(errors.NotInitializedRepoError).with.property('code', errors.NotInitializedRepoError.code) + await expect(version.getVersion(dir, repoOptions)).to.be.eventually.rejectedWith(errors.NotInitializedRepoError).with.property('code', errors.NotInitializedRepoError.code) // Create version file - const store = new Datastore(dir, { extension: '', createIfMissing: false }) + const { + StorageBackend, + storageOptions + } = getDatastoreAndOptions(repoOptions, 'root') + + const store = new StorageBackend(dir, { + ...storageOptions, + createIfMissing: false + }) + await store.open() - await store.put(new Key('config'), Buffer.from('some dummy config')) - await store.put(new Key('version'), Buffer.from('5')) + await store.put(CONFIG_KEY, Buffer.from('some dummy config')) + await store.put(VERSION_KEY, Buffer.from('5')) await store.close() - await version.setVersion(dir, 7) - expect(await version.getVersion(dir)).to.be.equal(7) + await version.setVersion(dir, 7, repoOptions) + expect(await version.getVersion(dir, repoOptions)).to.be.equal(7) }) }) } From 2940dac07b5968c1e099adc94ba353b995774040 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 6 Aug 2020 12:52:01 +0100 Subject: [PATCH 2/4] chore: remove cli as it is not used anywhere --- .travis.yml | 7 --- package.json | 7 +-- src/cli.js | 62 --------------------------- test/browser.js | 53 +++-------------------- test/fixtures/repo.js | 49 +++++++++++++++++++++ test/node.js | 54 +++-------------------- test/sharness/.gitignore | 5 --- test/sharness/Makefile | 53 ----------------------- test/sharness/lib/install-sharness.sh | 60 -------------------------- test/sharness/lib/test-lib.sh | 34 --------------- test/sharness/t0000-sharness.sh | 32 -------------- test/sharness/t0010-basic-commands.sh | 55 ------------------------ 12 files changed, 63 insertions(+), 408 deletions(-) delete mode 100755 src/cli.js create mode 100644 test/fixtures/repo.js delete mode 100644 test/sharness/.gitignore delete mode 100644 test/sharness/Makefile delete mode 100755 test/sharness/lib/install-sharness.sh delete mode 100644 test/sharness/lib/test-lib.sh delete mode 100755 test/sharness/t0000-sharness.sh delete mode 100755 test/sharness/t0010-basic-commands.sh diff --git a/.travis.yml b/.travis.yml index a6f125a..cca74cb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,13 +37,6 @@ jobs: firefox: latest script: npx aegir test -t browser -- --browsers FirefoxHeadless - - stage: test - name: sharness - os: - - linux - - osx - script: cd ./test/sharness && make - - stage: test name: electron-main os: osx diff --git a/package.json b/package.json index 8a019da..7602e23 100644 --- a/package.json +++ b/package.json @@ -23,9 +23,6 @@ "./src/repo/default-root-options.js": "./src/repo/ldefault-root-options.browser.js", "datastore-fs": "datastore-level" }, - "bin": { - "jsipfs-migrations": "./src/cli.js" - }, "repository": { "type": "git", "url": "https://github.com/ipfs/js-ipfs-repo-migrations.git" @@ -54,9 +51,7 @@ "debug": "^4.1.0", "interface-datastore": "^1.0.2", "multibase": "^1.0.1", - "proper-lockfile": "^4.1.1", - "yargs": "^15.3.1", - "yargs-promise": "^1.1.0" + "proper-lockfile": "^4.1.1" }, "devDependencies": { "aegir": "^23.0.0", diff --git a/src/cli.js b/src/cli.js deleted file mode 100755 index 102c43c..0000000 --- a/src/cli.js +++ /dev/null @@ -1,62 +0,0 @@ -#! /usr/bin/env node - -'use strict' - -const YargsPromise = require('yargs-promise') -const yargs = require('yargs') -const process = require('process') -const log = require('debug')('repo-migrations:cli') - -const commands = require('./commands') - -function print (msg = '', newline = true) { - msg = newline ? msg + '\n' : msg - process.stdout.write(msg) -} - -async function main (args) { - const cli = yargs() - .option('repo-path', { - desc: 'Path to the IPFS repo', - type: 'string' - }) - .option('migrations', { - desc: 'Path to folder with migrations. Default: bundled migrations', - type: 'string' - }) - .command(commands.migrate) - .command(commands.status) - .command(commands.add) - .demandCommand(1, 'You need at least one command before continuing') - .strict() - .fail((msg, err, yargs) => { - if (err) { - throw err // preserve stack - } - - // Suppress "one command required" error when only help page is to be displayed - if (args.length > 0) { - print(msg) - } - - yargs.showHelp() - }) - - try { - const { data } = await new YargsPromise(cli).parse(args) - if (data) print(data) - } catch (err) { - log(err) - - // the argument can have a different shape depending on where the error came from - if (err.message || (err.error && err.error.message)) { - print(err.message || err.error.message) - } else { - print('Unknown error, please re-run the command with DEBUG=repo-migrations:cli to see debug output') - } - - process.exit(1) - } -} - -main(process.argv.slice(2)) diff --git a/test/browser.js b/test/browser.js index 58d7da0..e074dbf 100644 --- a/test/browser.js +++ b/test/browser.js @@ -1,9 +1,7 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') -const loadFixture = require('aegir/fixtures') -const { CONFIG_KEY, VERSION_KEY, getDatastoreAndOptions } = require('../src/utils') +const { createRepo, createAndLoadRepo } = require('./fixtures/repo') const repoOptions = { lock: 'memory', @@ -38,68 +36,29 @@ const repoOptions = { } } -async function createRepo () { - const { - StorageBackend, - storageOptions - } = getDatastoreAndOptions(repoOptions, 'root') - - const date = Date.now().toString() - const dir = 'test-repo-for-' + date - const store = new StorageBackend(dir, { - ...storageOptions, - createIfMissing: true - }) - await store.open() - await store.close() - return dir -} - -async function createAndLoadRepo () { - const { - StorageBackend, - storageOptions - } = getDatastoreAndOptions(repoOptions, 'root') - - const date = Date.now().toString() - const dir = 'test-repo-for-' + date - const store = new StorageBackend(dir, { - ...storageOptions, - createIfMissing: true - }) - await store.open() - - await store.put(VERSION_KEY, Buffer.from(loadFixture('test/fixtures/test-repo/version'))) - await store.put(CONFIG_KEY, Buffer.from(loadFixture('test/fixtures/test-repo/config'))) - - await store.close() - - return dir -} - async function repoCleanup (dir) { } describe('Browser specific tests', () => { describe('lock.js tests', () => { describe('mem-lock tests', () => { - require('./lock-test')(require('../src/repo/lock-memory'), createRepo, repoCleanup, repoOptions) + require('./lock-test')(require('../src/repo/lock-memory'), () => createRepo(repoOptions), repoCleanup, repoOptions) }) }) describe('version tests', () => { - require('./version-test')(createRepo, repoCleanup, repoOptions) + require('./version-test')(() => createRepo(repoOptions), repoCleanup, repoOptions) }) describe('migrations tests', () => { - require('./migrations/migration-8-test')(createRepo, repoCleanup, repoOptions) + require('./migrations/migration-8-test')(() => createRepo(repoOptions), repoCleanup, repoOptions) }) describe('init tests', () => { - require('./init-test')(createRepo, repoCleanup, repoOptions) + require('./init-test')(() => createRepo(repoOptions), repoCleanup, repoOptions) }) describe('integration tests', () => { - require('./integration-test')(createAndLoadRepo, repoCleanup, repoOptions) + require('./integration-test')(() => createAndLoadRepo(repoOptions), repoCleanup, repoOptions) }) }) diff --git a/test/fixtures/repo.js b/test/fixtures/repo.js new file mode 100644 index 0000000..5e19f44 --- /dev/null +++ b/test/fixtures/repo.js @@ -0,0 +1,49 @@ +'use strict' + +const { Buffer } = require('buffer') +const loadFixture = require('aegir/fixtures') +const { CONFIG_KEY, VERSION_KEY, getDatastoreAndOptions } = require('../../src/utils') + +async function createRepo (repoOptions) { + const { + StorageBackend, + storageOptions + } = getDatastoreAndOptions(repoOptions, 'root') + + const date = Date.now().toString() + const dir = 'test-repo-for-' + date + const store = new StorageBackend(dir, { + ...storageOptions, + createIfMissing: true + }) + await store.open() + await store.close() + return dir +} + +async function createAndLoadRepo (repoOptions) { + const { + StorageBackend, + storageOptions + } = getDatastoreAndOptions(repoOptions, 'root') + + const date = Date.now().toString() + const dir = 'test-repo-for-' + date + const store = new StorageBackend(dir, { + ...storageOptions, + createIfMissing: true + }) + await store.open() + + await store.put(VERSION_KEY, Buffer.from(loadFixture('test/fixtures/test-repo/version'))) + await store.put(CONFIG_KEY, Buffer.from(loadFixture('test/fixtures/test-repo/config'))) + + await store.close() + + return dir +} + +module.exports = { + createRepo, + createAndLoadRepo +} diff --git a/test/node.js b/test/node.js index b02c2d6..eeb0636 100644 --- a/test/node.js +++ b/test/node.js @@ -3,8 +3,7 @@ const promisify = require('util').promisify const asyncRimraf = promisify(require('rimraf')) -const loadFixture = require('aegir/fixtures') -const { CONFIG_KEY, VERSION_KEY, getDatastoreAndOptions } = require('../src/utils') +const { createRepo, createAndLoadRepo } = require('./fixtures/repo') const repoOptions = { lock: 'fs', @@ -28,45 +27,6 @@ const repoOptions = { } } -async function createRepo () { - const { - StorageBackend, - storageOptions - } = getDatastoreAndOptions(repoOptions, 'root') - - const date = Date.now().toString() - const dir = 'test-repo-for-' + date - const store = new StorageBackend(dir, { - ...storageOptions, - createIfMissing: true - }) - await store.open() - await store.close() - return dir -} - -async function createAndLoadRepo () { - const { - StorageBackend, - storageOptions - } = getDatastoreAndOptions(repoOptions, 'root') - - const date = Date.now().toString() - const dir = 'test-repo-for-' + date - const store = new StorageBackend(dir, { - ...storageOptions, - createIfMissing: true - }) - await store.open() - - await store.put(VERSION_KEY, Buffer.from(loadFixture('test/fixtures/test-repo/version'))) - await store.put(CONFIG_KEY, Buffer.from(loadFixture('test/fixtures/test-repo/config'))) - - await store.close() - - return dir -} - function repoCleanup (dir) { return asyncRimraf(dir) } @@ -74,27 +34,27 @@ function repoCleanup (dir) { describe('Node specific tests', () => { describe('lock.js tests', () => { describe('fs-lock tests', () => { - require('./lock-test')(require('../src/repo/lock'), createRepo, repoCleanup, repoOptions) + require('./lock-test')(require('../src/repo/lock'), () => createRepo(repoOptions), repoCleanup, repoOptions) }) describe('mem-lock tests', () => { - require('./lock-test')(require('../src/repo/lock-memory'), createRepo, repoCleanup, repoOptions) + require('./lock-test')(require('../src/repo/lock-memory'), () => createRepo(repoOptions), repoCleanup, repoOptions) }) }) describe('version tests', () => { - require('./version-test')(createRepo, repoCleanup, repoOptions) + require('./version-test')(() => createRepo(repoOptions), repoCleanup, repoOptions) }) describe('migrations tests', () => { - require('./migrations/migration-8-test')(createRepo, repoCleanup, repoOptions) + require('./migrations/migration-8-test')(() => createRepo(repoOptions), repoCleanup, repoOptions) }) describe('init tests', () => { - require('./init-test')(createRepo, repoCleanup, repoOptions) + require('./init-test')(() => createRepo(repoOptions), repoCleanup, repoOptions) }) describe('integration tests', () => { - require('./integration-test')(createAndLoadRepo, repoCleanup, repoOptions) + require('./integration-test')(() => createAndLoadRepo(repoOptions), repoCleanup, repoOptions) }) }) diff --git a/test/sharness/.gitignore b/test/sharness/.gitignore deleted file mode 100644 index a0b371d..0000000 --- a/test/sharness/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -!/lib/ -bin/ -lib/sharness/ -test-results/ -trash directory.*.sh/ diff --git a/test/sharness/Makefile b/test/sharness/Makefile deleted file mode 100644 index 0eb78b5..0000000 --- a/test/sharness/Makefile +++ /dev/null @@ -1,53 +0,0 @@ -# Run sharness tests -# -# Copyright (c) 2016 Christian Couder -# MIT Licensed; see the LICENSE file in this repository. -# -# NOTE: run with TEST_VERBOSE=1 for verbose sharness tests. - -T = $(sort $(wildcard t[0-9][0-9][0-9][0-9]-*.sh)) -LIBDIR = lib -SHARNESSDIR = sharness -AGGREGATE = $(LIBDIR)/$(SHARNESSDIR)/aggregate-results.sh - - -BINS = ../bin/jsipfs-migrations - -all: aggregate - -help: - @echo "- use 'make' or 'make all' to run all the tests" - @echo "- use 'make deps' to create an 'ipfs' executable in ../bin" - @echo "- to run tests manually, make sure to include ../bin in your PATH" - -clean: clean-test-results - @echo "*** $@ ***" - -rm -rf ../bin/ipfs - -clean-test-results: - @echo "*** $@ ***" - -rm -rf test-results - -$(T): clean-test-results deps - @echo "*** $@ ***" - ./$@ - -aggregate: clean-test-results $(T) - @echo "*** $@ ***" - ls test-results/t*-*.sh.*.counts | $(AGGREGATE) - -deps: sharness $(BINS) curl - -sharness: - @echo "*** checking $@ ***" - lib/install-sharness.sh - -../bin/jsipfs-migrations: - mkdir -p bin - -ln -s ../../../src/cli.js ./bin/jsipfs-migrations - -curl: - @which curl >/dev/null || (echo "Please install curl!" && false) - -.PHONY: all help clean clean-test-results $(T) aggregate deps sharness - diff --git a/test/sharness/lib/install-sharness.sh b/test/sharness/lib/install-sharness.sh deleted file mode 100755 index ecc5834..0000000 --- a/test/sharness/lib/install-sharness.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/sh -# install-sharness.sh -# -# Copyright (c) 2014 Juan Batiz-Benet -# Copyright (c) 2015 Christian Couder -# MIT Licensed; see the LICENSE file in this repository. -# -# This script checks that Sharness is installed in: -# -# $(pwd)/$clonedir/$sharnessdir/ -# -# where $clonedir and $sharnessdir are configured below. -# -# If Sharness is not installed, this script will clone it -# from $urlprefix (defined below). -# -# If Sharness is not uptodate with $version (defined below), -# this script will fetch and will update the installed -# version to $version. -# - -# settings -version=ecba410b0b58400dd6517cfa6594fdac243d9056 -urlprefix=https://github.com/chriscool/sharness.git -clonedir=lib -sharnessdir=sharness - -if test -f "$clonedir/$sharnessdir/SHARNESS_VERSION_$version" -then - # There is the right version file. Great, we are done! - exit 0 -fi - -die() { - echo >&2 "$@" - exit 1 -} - -checkout_version() { - git checkout "$version" || die "Could not checkout '$version'" - rm -f SHARNESS_VERSION_* || die "Could not remove 'SHARNESS_VERSION_*'" - touch "SHARNESS_VERSION_$version" || die "Could not create 'SHARNESS_VERSION_$version'" - echo "Sharness version $version is checked out!" -} - -if test -d "$clonedir/$sharnessdir/.git" -then - # We need to update sharness! - cd "$clonedir/$sharnessdir" || die "Could not cd into '$clonedir/$sharnessdir' directory" - git fetch || die "Could not fetch to update sharness" -else - # We need to clone sharness! - mkdir -p "$clonedir" || die "Could not create '$clonedir' directory" - cd "$clonedir" || die "Could not cd into '$clonedir' directory" - - git clone "$urlprefix" || die "Could not clone '$urlprefix'" - cd "$sharnessdir" || die "Could not cd into '$sharnessdir' directory" -fi - -checkout_version diff --git a/test/sharness/lib/test-lib.sh b/test/sharness/lib/test-lib.sh deleted file mode 100644 index 7c0f309..0000000 --- a/test/sharness/lib/test-lib.sh +++ /dev/null @@ -1,34 +0,0 @@ -# Test framework for go-ipfs -# -# Copyright (c) 2014 Christian Couder -# MIT Licensed; see the LICENSE file in this repository. -# -# We are using sharness (https://github.com/chriscool/sharness) -# which was extracted from the Git test framework. - -# add current directory to path. -BIN=$(echo `pwd`/bin) -PATH=${BIN}:${PATH} - -# assert the `jsipfs-migrations` we're using is the right one. -if [[ $(which jsipfs-migrations) != ${BIN}/jsipfs-migrations ]]; then - echo >&2 "Cannot find the test's local jsipfs-migrations tool." - echo >&2 "Please check test and ipfs tool installation." - JS_BIN=$(dirname $(dirname "${BIN}"))"/src/cli.js" - echo >&2 "For jsipfs-migrations, look for a symlink from '${BIN}/jsipfs-migrations' to '${JS_BIN}'." - echo >&2 "Use 'make' or 'make deps' as it should install this symlink." - exit 1 -fi - -# set sharness verbosity. we set the env var directly as -# it's too late to pass in --verbose, and --verbose is harder -# to pass through in some cases. -test "$TEST_VERBOSE" = 1 && verbose=t - -SHARNESS_LIB="lib/sharness/sharness.sh" - -. "$SHARNESS_LIB" || { - echo >&2 "Cannot source: $SHARNESS_LIB" - echo >&2 "Please check Sharness installation." - exit 1 -} diff --git a/test/sharness/t0000-sharness.sh b/test/sharness/t0000-sharness.sh deleted file mode 100755 index c55d40e..0000000 --- a/test/sharness/t0000-sharness.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh - -test_description="Show basic features of Sharness" - -. ./lib/sharness/sharness.sh - -test_expect_success "Success is reported like this" " - echo hello world | grep hello -" - -test_expect_success "Commands are chained this way" " - test x = 'x' && - test 2 -gt 1 && - echo success -" - -return_42() { - echo "Will return soon" - return 42 -} - -test_expect_success "You can test for a specific exit code" " - test_expect_code 42 return_42 -" - -test_expect_failure "We expect this to fail" " - test 1 = 2 -" - -test_done - -# vi: set ft=sh : diff --git a/test/sharness/t0010-basic-commands.sh b/test/sharness/t0010-basic-commands.sh deleted file mode 100755 index c3ee372..0000000 --- a/test/sharness/t0010-basic-commands.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/sh -# -# Copyright (c) 2014 Christian Couder -# MIT Licensed; see the LICENSE file in this repository. -# - -test_description="Test installation and some basic commands" - -# setup temp repo -rm -rf ./tmp -cp -r ../fixtures/test-repo ./tmp -IPFS_PATH=$(echo `pwd`/tmp) -export IPFS_PATH - -. lib/test-lib.sh - -test_expect_success "current dir is writable" ' - echo "It works!" > test.txt -' - -test_expect_success "jsipfs-migrations version succeeds" ' - jsipfs-migrations --version > version.txt -' - -test_expect_success "jsipfs-migrations help succeeds" ' - jsipfs-migrations --help > help.txt && - grep "migrate" help.txt -' - -test_expect_success "jsipfs-migrations status shows migrations are needed" $' - jsipfs-migrations status --migrations ../test/test-migrations > status.txt && - grep "Repo is out of date" status.txt && - grep "Current repo version: 1" status.txt && - grep "Latest migration version: 2" status.txt -' - -test_expect_success "jsipfs-migrations successfully migrate to latest version" $' - jsipfs-migrations migrate --migrations ../test/test-migrations > migrate.txt && - grep "Successfully migrated to version 2" migrate.txt -' - -test_expect_success "jsipfs-migrations status shows NO migrations are needed" $' - jsipfs-migrations status --migrations ../test/test-migrations > status.txt && - grep "Nothing to migrate" status.txt && - grep "Current repo version: 2" status.txt && - grep "Latest migration version: 2" status.txt -' - -test_expect_success "jsipfs-migrations successfully reverts" $' - jsipfs-migrations migrate --to 1 --revert-ok --migrations ../test/test-migrations > revert.txt && - grep "Successfully reverted version 2" revert.txt -' - - -test_done From 954b447aa08653232474a7e9d446f550982c7efd Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 6 Aug 2020 14:11:52 +0100 Subject: [PATCH 3/4] chore: pr comments --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f82df1e..e8c5bf1 100644 --- a/README.md +++ b/README.md @@ -116,9 +116,9 @@ Executes a forward migration to a specific version, or to the latest version if * `path` (string, mandatory) - path to the repo to be migrated * `toVersion` (int, mandatory) - version to which the repo should be migrated. - * `options` (object, optional) - options for the migration + * `options` (object, mandatory) - options for the migration * `options.ignoreLock` (bool, optional) - if true will not lock the repo when applying migrations. Use with caution. - * `options.repoOptions` (object, optional) - options that are passed to migrations, that use them to construct the datastore. (options are the same as for IPFSRepo). + * `options.repoOptions` (object, mandatory) - options that are passed to migrations, that use them to construct the datastore. (options are the same as for IPFSRepo). * `options.onProgress` (function, optional) - callback that is called after finishing execution of each migration to report progress. * `options.isDryRun` (bool, optional) - flag that indicates if it is a dry run that should give the same output as running a migration but without making any actual changes. @@ -139,9 +139,9 @@ Executes backward migration to a specific version. * `path` (string, mandatory) - path to the repo to be reverted * `toVersion` (int, mandatory) - version to which the repo should be reverted to. - * `options` (object, optional) - options for the reversion + * `options` (object, mandatory) - options for the reversion * `options.ignoreLock` (bool, optional) - if true will not lock the repo when applying migrations. Use with caution. - * `options.repoOptions` (object, optional) - options that are passed to migrations, that use them to construct the datastore. (options are the same as for IPFSRepo). + * `options.repoOptions` (object, mandatory) - options that are passed to migrations, that use them to construct the datastore. (options are the same as for IPFSRepo). * `options.onProgress` (function, optional) - callback that is called after finishing execution of each migration to report progress. * `options.isDryRun` (bool, optional) - flag that indicates if it is a dry run that should give the same output as running a migration but without making any actual changes. From 47b43caebc3eff60489575c7cdfc1b95f1da4e4c Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 6 Aug 2020 14:23:00 +0100 Subject: [PATCH 4/4] chore: remove extraneous file and dep --- package.json | 1 - src/commands.js | 165 ------------------------------------------------ 2 files changed, 166 deletions(-) delete mode 100644 src/commands.js diff --git a/package.json b/package.json index 7602e23..e52ff8a 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,6 @@ }, "dependencies": { "buffer": "^5.6.0", - "chalk": "^4.0.0", "cids": "^0.8.3", "datastore-core": "^1.1.0", "datastore-fs": "^1.0.0", diff --git a/src/commands.js b/src/commands.js deleted file mode 100644 index 04a23e9..0000000 --- a/src/commands.js +++ /dev/null @@ -1,165 +0,0 @@ -'use strict' - -const os = require('os') -const path = require('path') -const fs = require('fs') -const process = require('process') -const util = require('util') -const readline = require('readline') - -const writeFile = util.promisify(fs.writeFile) -const mkdir = util.promisify(fs.mkdir) - -const chalk = require('chalk') - -const repoVersion = require('./repo/version') -const migratorUtils = require('./utils') -const migrator = require('./index') -const templates = require('./migration-templates') -const migrations = require('../migrations') - -function shouldContinueWithIrreversibleMigration () { - return new Promise((resolve, reject) => { - const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout - }) - - rl.question(chalk.yellow('There is irreversible migration in the migrations plan. Do you want to continue? [y/N]'), (answer) => { - rl.close() - resolve(answer === 'y') - }) - }) -} - -function asyncClosure (fnc) { - return function asyncWrapper ({ resolve, ...options }) { - resolve(fnc(options)) - } -} - -function reportingClosure (action) { - return (migration, currentlyMigrated, totalToMigrate) => - process.stdout.write(`${chalk.green(`[${currentlyMigrated}/${totalToMigrate}]`)} Successfully ${action} ${chalk.bold(migration.version)}: ${migration.description}\n`) -} - -async function migrate ({ repoPath, repoOptions, migrations, to, dry, revertOk }) { - repoPath = repoPath || process.env.IPFS_PATH || path.join(os.homedir(), '.jsipfs') - migrations = migrations === undefined ? require('../migrations') : require(migrations) - - if (!to) { - to = migrator.getLatestMigrationVersion(migrations) - } - - const version = await repoVersion.getVersion(repoPath, repoOptions) - - let action - if (dry) { - action = 'loaded migration' - } else { - if (!to || (version <= to)) { - action = 'migrated to version' - } else { - action = 'reverted version' - } - } - - const options = { - migrations: migrations, - toVersion: to, - ignoreLock: false, - onProgress: reportingClosure(action), - isDryRun: dry - } - - if (version === to) { - return chalk.yellow('Nothing to migrate! Versions matches') - } else if (version < to) { - if (migratorUtils.containsIrreversibleMigration(version, to, migrations)) { - if (!(await shouldContinueWithIrreversibleMigration())) { - return 'Migration aborted' - } - } - - await migrator.migrate(repoPath, to, options) - } else if (revertOk) { - await migrator.revert(repoPath, to, options) - } else { - throw new Error(`The migration would revert the repo version from ${version} to ${to}. To revert you must supply '--revert-ok'.`) - } -} - -async function status ({ repoPath, migrations }) { - repoPath = repoPath || process.env.IPFS_PATH || path.join(os.homedir(), '.jsipfs') - - // Import migrations if set - if (migrations) { - migrations = require(migrations) - } - - const version = await repoVersion.getVersion(repoPath) - const lastMigrationVersion = migrator.getLatestMigrationVersion(migrations) - const statusString = - version < lastMigrationVersion ? chalk.yellow('Repo is out of date') : chalk.green('Nothing to migrate') - - return `${statusString}\nCurrent repo version: ${version}\nLatest migration version: ${lastMigrationVersion}` -} - -async function add ({ empty }) { - const newMigrationVersion = migrator.getLatestMigrationVersion() + 1 - const newMigrationFolder = path.join(__dirname, '..', 'migrations', 'migration-' + newMigrationVersion) - - const migrationsImport = migrations.map((migration) => migration.empty ? ` Object.assign({version: ${parseInt(migration.version)}}, emptyMigration),` : ` require('./migration-${parseInt(migration.version)}'),`) - if (empty) { - migrationsImport.push(` Object.assign({version: ${newMigrationVersion}}, emptyMigration),`) - } else { - migrationsImport.push(` require('./migration-${newMigrationVersion}'),`) - } - const migrationsIndexJsContent = templates.migrationsIndexJs - .replace('{{imports}}', migrationsImport.join('\n')) - await writeFile(path.join(newMigrationFolder, '..', 'index.js'), migrationsIndexJsContent) - - if (empty) return - - await mkdir(newMigrationFolder) - - const indexJsContent = templates.indexJs - .replace(/{{version}}/gi, newMigrationVersion) - await writeFile(path.join(newMigrationFolder, 'index.js'), indexJsContent) -} - -module.exports = { - migrate: { - command: 'migrate', - describe: 'Migrate IPFS repo to latest or specific version', - handler: asyncClosure(migrate), - builder: yargv => yargv - .option('to', { - describe: 'Target version to migrate repo to', - type: 'number' - }) - .option('dry', { - describe: 'Output migration results but don\'t actually perform migration', - type: 'boolean' - }) - .option('revert-ok', { - describe: 'Allow migrating to a lower version (reverting)', - type: 'boolean' - }) - }, - status: { - command: 'status', - describe: 'Display migration status of IPFS repo', - handler: asyncClosure(status) - }, - add: { - command: 'add', - describe: 'Bootstrap new migration', - handler: asyncClosure(add), - builder: yargv => yargv - .option('empty', { - describe: 'Creates empty migration', - type: 'boolean' - }) - } -}