From cb45fadef0c5add229e4d3ec45b70ac9567cca6c Mon Sep 17 00:00:00 2001 From: Aschen Date: Wed, 16 Nov 2022 22:29:19 +0100 Subject: [PATCH 1/2] Add migrate command --- features/Collection.feature | 27 ++++++++++++ features/fixtures/mappings.js | 2 + features/fixtures/migration.js | 11 +++++ package-lock.json | 9 +++- package.json | 2 +- src/commands/collection/migrate.ts | 70 ++++++++++++++++++++++++++++++ src/support/restore-collection.ts | 16 +++++-- 7 files changed, 132 insertions(+), 5 deletions(-) create mode 100644 features/fixtures/migration.js create mode 100644 src/commands/collection/migrate.ts diff --git a/features/Collection.feature b/features/Collection.feature index 75bd71dc..2894151b 100644 --- a/features/Collection.feature +++ b/features/Collection.feature @@ -49,3 +49,30 @@ Feature: Collection Commands | collection | "yellow-taxi" | Then I should receive a result matching: | dynamic | "false" | + + # collection:migrate ====================================== + + @mappings + Scenario: Migrate a collection + Given an existing collection "nyc-open-data":"yellow-taxi" + Given I "create" the following documents: + | _id | body | + | "antoine" | { "name": "Antoine Ducuroy" } | + | "karina" | { "name": "Karina Tsimashenka" } | + Given I refresh the collection + Given I run the command "collection:export" with: + | arg | nyc-open-data | | + | arg | yellow-taxi | | + Given I successfully call the route "collection":"truncate" with args: + | index | "nyc-open-data" | + | collection | "yellow-taxi" | + # migrate + When I run the command "collection:migrate" with args: + | "./features/fixtures/migration.js" | + | "./nyc-open-data/yellow-taxi" | + Then The document "antoine" content match: + | firstName | "Antoine" | + | lastName | "Ducuroy" | + Then The document "karina" content match: + | firstName | "Karina" | + | lastName | "Tsimashenka" | diff --git a/features/fixtures/mappings.js b/features/fixtures/mappings.js index 0f846945..61c94bc5 100644 --- a/features/fixtures/mappings.js +++ b/features/fixtures/mappings.js @@ -5,6 +5,8 @@ module.exports = { properties: { name: { type: "keyword" }, city: { type: "keyword" }, + firstName: { type: "keyword" }, + lastName: { type: "keyword" } }, }, }, diff --git a/features/fixtures/migration.js b/features/fixtures/migration.js new file mode 100644 index 00000000..9197e07e --- /dev/null +++ b/features/fixtures/migration.js @@ -0,0 +1,11 @@ +function migrate(document) { + const [firstName, lastName] = document.body.name.split(' '); + + return { + _id: document._id, + body: { + firstName, + lastName, + } + }; +} \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 9883a3a5..d54a54d1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -55,7 +55,7 @@ "@typescript-eslint/eslint-plugin": "^4.22.0", "@typescript-eslint/parser": "^4.22.0", "chai": "^4.3.4", - "cucumber": "^6.0.5", + "cucumber": "^6.0.7", "eslint": "^7.24.0", "globby": "^11.0.3", "mocha": "^8.3.2", @@ -1916,6 +1916,7 @@ "version": "6.0.7", "resolved": "https://registry.npmjs.org/cucumber/-/cucumber-6.0.7.tgz", "integrity": "sha512-pN3AgWxHx8rOi+wOlqjASNETOjf3TgeyqhMNLQam7nSTXgQzju1oAmXkleRQRcXvpVvejcDHiZBLFSfBkqbYpA==", + "deprecated": "Cucumber is publishing new releases under @cucumber/cucumber", "dev": true, "dependencies": { "assertion-error-formatter": "^3.0.0", @@ -1945,6 +1946,12 @@ "title-case": "^2.1.1", "util-arity": "^1.0.2", "verror": "^1.9.0" + }, + "bin": { + "cucumber-js": "bin/cucumber-js" + }, + "engines": { + "node": ">=8" } }, "node_modules/cucumber-expressions": { diff --git a/package.json b/package.json index 8c24e025..c9e3f307 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,7 @@ "@typescript-eslint/eslint-plugin": "^4.22.0", "@typescript-eslint/parser": "^4.22.0", "chai": "^4.3.4", - "cucumber": "^6.0.5", + "cucumber": "^6.0.7", "eslint": "^7.24.0", "globby": "^11.0.3", "mocha": "^8.3.2", diff --git a/src/commands/collection/migrate.ts b/src/commands/collection/migrate.ts new file mode 100644 index 00000000..fc9e8a8b --- /dev/null +++ b/src/commands/collection/migrate.ts @@ -0,0 +1,70 @@ +import path from "path"; +import fs from "fs"; + +import { flags } from "@oclif/command"; + +import { Kommand } from "../../common"; +import { kuzzleFlags } from "../../support/kuzzle"; +import { restoreCollectionData } from "../../support/restore-collection"; +import { JSONObject } from "kuzzle-sdk"; + +export default class CollectionMigrate extends Kommand { + static keepAuth = true; + + static description = "Migrate a collection"; + + static flags = { + help: flags.help({}), + "batch-size": flags.string({ + description: "Maximum batch size (see limits.documentsWriteCount config)", + default: "200", + }), + index: flags.string({ + description: "If set, override the index destination name", + }), + collection: flags.string({ + description: "If set, override the collection destination name", + }), + ...kuzzleFlags, + protocol: flags.string({ + description: "Kuzzle protocol (http or websocket)", + default: "ws", + }), + }; + + static args = [ + { name: "script", description: "Migration script path", required: true }, + { name: "path", description: "Collection dump path", required: true }, + ]; + + async runSafe() { + this.logInfo(`Start migration documents with ${this.args.script} from ${this.args.path}`); + + const migrateDocument = this.getMigrationFunction(); + + const { index, collection, total } = await restoreCollectionData( + this.sdk, + this.log.bind(this), + Number(this.flags["batch-size"]), + path.join(this.args.path, "documents.jsonl"), + this.flags.index, + this.flags.collection, + migrateDocument + ); + + this.logOk( + `Successfully migrated ${total} documents from "${this.args.path}" in "${index}:${collection}"` + ); + } + + private getMigrationFunction(): (document: JSONObject) => JSONObject { + const migrationScript = fs.readFileSync(this.args.script, "utf8"); + + let migrationFunction; + { + migrationFunction = eval(`var f = ${migrationScript}; f`); + } + + return migrationFunction; + } +} diff --git a/src/support/restore-collection.ts b/src/support/restore-collection.ts index 01c1d503..d18bdc56 100644 --- a/src/support/restore-collection.ts +++ b/src/support/restore-collection.ts @@ -1,6 +1,8 @@ -import chalk from "chalk"; import fs from "fs"; + +import chalk from "chalk"; import ndjson from "ndjson"; +import { JSONObject } from "kuzzle-sdk"; function handleError(log: any, dumpFile: string, error: any) { if (error.status === 206) { @@ -34,7 +36,8 @@ export async function restoreCollectionData( batchSize: number, dumpFile: string, index?: string, - collection?: string + collection?: string, + migrateDocument?: (document: JSONObject) => JSONObject ) { const mWriteRequest = { controller: "bulk", @@ -57,7 +60,14 @@ export async function restoreCollectionData( .pipe(ndjson.parse()) .on("data", (obj: any) => { if (headerSkipped) { - documents.push(obj); + const document = migrateDocument ? migrateDocument(obj) : obj; + if (!document._id) { + throw new Error(`Document does not have an "_id" property: ${JSON.stringify(document)}`); + } + if (!document.body) { + throw new Error(`Document does not have an "body" property: ${JSON.stringify(document)}`); + } + documents.push(document); if (documents.length === batchSize) { mWriteRequest.body.documents = documents; From 40218f4870f77804fc848767cafb045504a23f3f Mon Sep 17 00:00:00 2001 From: Aschen Date: Thu, 29 Dec 2022 22:27:06 +0100 Subject: [PATCH 2/2] Skip "type" fields from mappings --- features/CSV.feature | 2 +- src/support/dump-collection.ts | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/features/CSV.feature b/features/CSV.feature index c505dbfc..37e29da1 100644 --- a/features/CSV.feature +++ b/features/CSV.feature @@ -45,7 +45,7 @@ Feature: CSV Export | flag | --format | csv | Then I get the file in "nyc-open-data/yellow-taxi/documents.csv" containing """ - _id,city,city.type,name,name.type + _id,city,firstName,lastName,name the-hive-th,changmai,,, luca,changmai,,, chuon-chuon-kim,hcmc,,, diff --git a/src/support/dump-collection.ts b/src/support/dump-collection.ts index 89798c3a..667183fa 100644 --- a/src/support/dump-collection.ts +++ b/src/support/dump-collection.ts @@ -83,14 +83,14 @@ abstract class AbstractDumper { * @returns void */ // eslint-disable-next-line @typescript-eslint/no-empty-function - public async setup() {} + public async setup() { } /** * One-shot call before iterating over the data. Can be * used to write the header of the dumped output. */ // eslint-disable-next-line @typescript-eslint/no-empty-function - public async writeHeader() {} + public async writeHeader() { } /** * You can put here the logic to write into the dump. @@ -111,7 +111,7 @@ abstract class AbstractDumper { abstract onResult(document: { _id: string; _source: any }): Promise; // eslint-disable-next-line @typescript-eslint/no-empty-function - public async tearDown() {} + public async tearDown() { } /** * The loop that iterates over the documents of the collection and @@ -254,6 +254,11 @@ class CSVDumper extends AbstractDumper { return; } this.fields = Object.keys(flattenObject(mappings.properties)); + for (const field of this.fields) { + if (field.endsWith(".type")) { + this.fields.splice(this.fields.indexOf(field), 1); + } + } } else { // Delete '_id' from the selected fields, since IDs are // _always_ exported.