Skip to content
This repository was archived by the owner on Oct 1, 2021. It is now read-only.

Commit 412a11d

Browse files
AuHauAuHau
committed
feat: migration 8
1 parent bc88612 commit 412a11d

File tree

8 files changed

+327
-5
lines changed

8 files changed

+327
-5
lines changed

‎migrations/index.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,5 @@ const emptyMigration = {
1111
}
1212

1313
module.exports = [
14+
require('./migration-8')
1415
]

‎migrations/migration-8/blocks-to-multihash.js‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
const path = require('path')
2+
const CID = require('cids')
3+
const Key = require('interface-datastore').Key
4+
const core = require('datastore-core')
5+
const ShardingStore = core.ShardingDatastore
6+
const base32 = require('base32.js')
7+
const utils = require('../../src/utils')
8+
const log = require('debug')('ipfs-repo-migrations:migration-8')
9+
10+
// This function in js-ipfs-repo defaults to not using sharding
11+
// but the default value of the options.sharding is True hence this
12+
// function defaults to use sharding.
13+
async function maybeWithSharding (filestore, options) {
14+
if (options.sharding === false) {
15+
return filestore
16+
}
17+
18+
const shard = new core.shard.NextToLast(2)
19+
return await ShardingStore.createOrOpen(filestore, shard)
20+
}
21+
22+
function keyToMultihash(key){
23+
// Key to CID
24+
const decoder = new base32.Decoder()
25+
const buff = decoder.finalize(key.toString().slice(1))
26+
const cid = new CID(Buffer.from(buff))
27+
28+
// CID to multihash
29+
const enc = new base32.Encoder()
30+
return new Key('/' + enc.finalize(cid.multihash), false)
31+
}
32+
33+
function keyToCid(key){
34+
// Key to CID
35+
const decoder = new base32.Decoder()
36+
const buff = decoder.write(key.toString().slice(1)).finalize()
37+
const cid = new CID(1, 'raw', Buffer.from(buff))
38+
39+
// CID to Key
40+
const enc = new base32.Encoder()
41+
return new Key('/' + enc.finalize(cid.buffer), false)
42+
}
43+
44+
async function process(repoPath, options, keyFunction){
45+
const { StorageBackend, storageOptions } = utils.getDatastoreAndOptions(options, 'blocks')
46+
47+
const baseStore = new StorageBackend(path.join(repoPath, 'blocks'), storageOptions)
48+
const store = await maybeWithSharding(baseStore, storageOptions)
49+
50+
try {
51+
const batch = store.batch()
52+
let counter = 0
53+
for await (const block of store.query({})) {
54+
const newKey = keyFunction(block.key)
55+
56+
// If the Key is CIDv0 then it is raw multihash and nothing is changing
57+
if(newKey.toString() !== block.key.toString()){
58+
counter += 1
59+
60+
log(`Migrating Block from ${block.key.toString()} to ${newKey.toString()}`)
61+
batch.delete(block.key)
62+
batch.put(newKey, block.value)
63+
}
64+
}
65+
66+
log(`Changing ${ counter } blocks`)
67+
await batch.commit()
68+
} finally {
69+
await store.close()
70+
}
71+
}
72+
73+
exports.migrate = function blocksMigrate (repoPath, options) {
74+
return process(repoPath, options, keyToMultihash)
75+
}
76+
77+
exports.revert = function blocksRevert (repoPath, options) {
78+
return process(repoPath, options, keyToCid)
79+
}

‎migrations/migration-8/index.js‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
'use strict'
2+
3+
const keysEncoding = require('./keys-encoding')
4+
const blocksToMultihash = require('./blocks-to-multihash')
5+
const log = require('debug')('ipfs-repo-migrations:migration-8')
6+
7+
async function migrate (repoPath, options) {
8+
await keysEncoding.migrate(repoPath, options)
9+
10+
try{
11+
await blocksToMultihash.migrate(repoPath, options)
12+
}catch (e) {
13+
log('During migration of Blockstore to multihash exception was raised! Reverting keys part of migration!')
14+
await keysEncoding.revert(repoPath, options)
15+
16+
throw e
17+
}
18+
}
19+
20+
async function revert (repoPath, options) {
21+
await keysEncoding.revert(repoPath, options)
22+
23+
try{
24+
await blocksToMultihash.revert(repoPath, options)
25+
}catch (e) {
26+
log('During reversion of Blockstore to CID exception was raised! Migrating keys part of migration!')
27+
await keysEncoding.migrate(repoPath, options)
28+
29+
throw e
30+
}
31+
}
32+
33+
module.exports = {
34+
version: 8,
35+
description: 'Transforms key\'s names into base32 encoding and converts Block store to use multihashes',
36+
migrate,
37+
revert
38+
}

‎migrations/migration-8/keys-encoding.js‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
const utils = require('../../src/utils')
2+
const path = require('path')
3+
const base32 = require('base32.js')
4+
const Key = require('interface-datastore').Key
5+
const log = require('debug')('ipfs-repo-migrations:migration-8')
6+
7+
const KEY_PREFIX = 'key_'
8+
9+
function encode (name) {
10+
name = Buffer.from(name)
11+
const encoder = new base32.Encoder({ type: 'rfc4648' })
12+
return (KEY_PREFIX + encoder.finalize(name)).toLowerCase()
13+
}
14+
15+
function decode (name) {
16+
if (!name.startsWith(KEY_PREFIX)) {
17+
throw Error('Unknown format of key\'s name!')
18+
}
19+
20+
const decoder = new base32.Decoder({ type: 'rfc4648' })
21+
const decodedNameBuff = decoder.finalize(name.replace(KEY_PREFIX, '').toUpperCase())
22+
return Buffer.from(decodedNameBuff).toString()
23+
}
24+
25+
async function processFolder (store, prefix, fileNameProcessor) {
26+
const query = {
27+
prefix: `/${ prefix }`
28+
}
29+
30+
const files = store.query(query)
31+
for await (let file of files) {
32+
const name = String(file.key._buf).replace(`/${ prefix }/`, '')
33+
const encodedFileName = fileNameProcessor(name)
34+
const newKey = new Key(`${ prefix }/${ encodedFileName }`)
35+
36+
await store.delete(file.key)
37+
log(`Translating key's name '${ file.key }' into '${ newKey }'`)
38+
await store.put(newKey, file.value)
39+
}
40+
}
41+
42+
async function process (repoPath, options, processor) {
43+
const { StorageBackend, storageOptions } = utils.getDatastoreAndOptions(options, 'keys')
44+
45+
const store = new StorageBackend(path.join(repoPath, 'keys'), storageOptions)
46+
try {
47+
const info = processFolder(store, 'info', processor)
48+
const data = processFolder(store, 'pkcs8', processor)
49+
50+
return await Promise.all([info, data])
51+
} finally {
52+
await store.close()
53+
}
54+
}
55+
56+
exports.migrate = async function keyEncode (repoPath, options) {
57+
return process(repoPath, options, encode)
58+
}
59+
60+
exports.revert = async function keyDecode (repoPath, options) {
61+
return process(repoPath, options, decode)
62+
}

‎package.json‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -46,13 +46,16 @@
4646
"docs": "aegir docs"
4747
},
4848
"dependencies": {
49+
"base32.js": "~0.1.0",
4950
"chalk": "^2.4.2",
50-
"datastore-fs": "~0.9.1",
51-
"datastore-level": "~0.12.1",
51+
"cids": "~0.7.0",
52+
"datastore-core": "~0.7.0",
53+
"datastore-fs": "~0.9.0",
54+
"datastore-level": "~0.12.0",
5255
"debug": "^4.1.0",
53-
"interface-datastore": "~0.8.0",
54-
"proper-lockfile": "^4.1.1",
55-
"yargs": "^14.2.0",
56+
"interface-datastore": "~0.7.0",
57+
"proper-lockfile": "^3.2.0",
58+
"yargs": "^12.0.5",
5659
"yargs-promise": "^1.1.0"
5760
},
5861
"devDependencies": {

‎test/browser.js‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,10 @@ describe('Browser specific tests', () => {
4343
require('./version-test')(createRepo, repoCleanup)
4444
})
4545

46+
describe('migrations tests', () => {
47+
require('./migrations/migration-8-test')(createRepo, repoCleanup)
48+
})
49+
4650
describe('init tests', () => {
4751
require('./init-test')(createRepo, repoCleanup)
4852
})

‎test/migrations/migration-8-test.js‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
/* eslint-env mocha */
2+
'use strict'
3+
4+
const chai = require('chai')
5+
chai.use(require('dirty-chai'))
6+
const chaiAsPromised = require('chai-as-promised')
7+
chai.use(chaiAsPromised)
8+
const expect = chai.expect
9+
10+
const path = require('path')
11+
const keysMigration = require('../../migrations/migration-8/keys-encoding')
12+
const blocksMigration = require('../../migrations/migration-8/blocks-to-multihash')
13+
const Key = require('interface-datastore').Key
14+
const Datastore = require('datastore-fs')
15+
const core = require('datastore-core')
16+
const ShardingStore = core.ShardingDatastore
17+
18+
const keysFixtures = [
19+
['aAa', 'key_mfawc'],
20+
['bbb', 'key_mjrge'],
21+
['self', 'key_onswyzq']
22+
]
23+
24+
const blocksFixtures = [
25+
['AFKREIBFG77IKIKDMBDUFDCSPK7H5TE5LNPMCSXYLPML27WSTT5YA5IUNU', 'CIQCKN76QUQUGYCHIKGFE6V6P3GJ2W26YFFPQW6YXV7NFHH3QB2RI3I']
26+
]
27+
28+
async function bootstrapKeys (dir, encoded) {
29+
const store = new Datastore(path.join(dir, 'keys'), { extension: '.data', createIfMissing: true })
30+
await store.open()
31+
32+
let name
33+
for (const keyNames of keysFixtures) {
34+
name = encoded ? keyNames[1] : keyNames[0]
35+
await store.put(new Key(`/pkcs8/${name}`), '')
36+
await store.put(new Key(`/info/${name}`), '')
37+
}
38+
39+
await store.close()
40+
}
41+
42+
async function validateKeys (dir, shouldBeEncoded) {
43+
const store = new Datastore(path.join(dir, 'keys'), { extension: '.data', createIfMissing: false })
44+
await store.open()
45+
46+
let name
47+
for (const keyNames of keysFixtures) {
48+
name = shouldBeEncoded ? keyNames[1] : keyNames[0]
49+
expect(await store.has(new Key(`/pkcs8/${name}`))).to.be.true(name)
50+
expect(await store.has(new Key(`/info/${name}`))).to.be.true(name)
51+
}
52+
53+
await store.close()
54+
}
55+
56+
async function bootstrapBlocks (dir, encoded) {
57+
const baseStore = new Datastore(path.join(dir, 'blocks'), { extension: '.data', createIfMissing: true })
58+
const shard = new core.shard.NextToLast(2)
59+
const store = await ShardingStore.createOrOpen(baseStore, shard)
60+
61+
let name
62+
for (const blocksNames of blocksFixtures) {
63+
name = encoded ? blocksNames[1] : blocksNames[0]
64+
await store.put(new Key(name), '')
65+
}
66+
67+
await store.close()
68+
}
69+
70+
async function validateBlocks (dir, shouldBeEncoded) {
71+
const baseStore = new Datastore(path.join(dir, 'blocks'), { extension: '.data', createIfMissing: false })
72+
const shard = new core.shard.NextToLast(2)
73+
const store = await ShardingStore.createOrOpen(baseStore, shard)
74+
75+
let newName, oldName
76+
for (const blockNames of blocksFixtures) {
77+
newName = shouldBeEncoded ? blockNames[1] : blockNames[0]
78+
oldName = shouldBeEncoded ? blockNames[0] : blockNames[1]
79+
expect(await store.has(new Key(oldName))).to.be.false(oldName)
80+
expect(await store.has(new Key(newName))).to.be.true(newName)
81+
}
82+
83+
await store.close()
84+
}
85+
86+
module.exports = (setup, cleanup) => {
87+
describe('migration 8', () => {
88+
let dir
89+
90+
beforeEach(async () => {
91+
dir = await setup()
92+
})
93+
afterEach(() => cleanup(dir))
94+
95+
it('should migrate keys forward', async () => {
96+
await bootstrapKeys(dir, false)
97+
await keysMigration.migrate(dir)
98+
await validateKeys(dir, true)
99+
})
100+
101+
it('should migrate keys backward', async () => {
102+
await bootstrapKeys(dir, true)
103+
await keysMigration.revert(dir)
104+
await validateKeys(dir, false)
105+
})
106+
107+
it('should fail to migrate keys backward with invalid key name', async () => {
108+
const store = new Datastore(path.join(dir, 'keys'), { extension: '.data', createIfMissing: true })
109+
await store.open()
110+
111+
await store.put(new Key('/pkcs8/mfawc'), '')
112+
await store.put(new Key('/info/mfawc'), '')
113+
114+
await store.close()
115+
116+
expect(keysMigration.revert(dir)).to.eventually.rejectedWith('Unknown format of key\'s name!')
117+
})
118+
119+
it('should migrate blocks forward', async () => {
120+
await bootstrapBlocks(dir, false)
121+
await blocksMigration.migrate(dir)
122+
await validateBlocks(dir, true)
123+
})
124+
//
125+
// it('should migrate blocks backward', async () => {
126+
// await bootstrapKeys(dir, true)
127+
// await blocksMigration.revert(dir)
128+
// await validateKeys(dir, false)
129+
// })
130+
})
131+
}

‎test/node.js‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,10 @@ describe('Node specific tests', () => {
4343
require('./version-test')(createRepo, repoCleanup)
4444
})
4545

46+
describe('migrations tests', () => {
47+
require('./migrations/migration-8-test')(createRepo, repoCleanup)
48+
})
49+
4650
describe('init tests', () => {
4751
require('./init-test')(createRepo, repoCleanup)
4852
})

0 commit comments

Comments
 (0)