Merge pull request #168 from kuzzleio/feat/migrate

Add `collection:migrate` command

features/CSV.feature

@@ -45,7 +45,7 @@ Feature: CSV Export
       | flag | --format      | csv |
     Then I get the file in "nyc-open-data/yellow-taxi/documents.csv" containing
       """
-      _id,city,city.type,name,name.type
+      _id,city,firstName,lastName,name
       the-hive-th,changmai,,,
       luca,changmai,,,
       chuon-chuon-kim,hcmc,,,

features/Collection.feature

@@ -49,3 +49,30 @@ Feature: Collection Commands
       | collection | "yellow-taxi"   |
     Then I should receive a result matching:
       | dynamic | "false" |
+
+  # collection:migrate ======================================
+
+  @mappings
+  Scenario: Migrate a collection
+    Given an existing collection "nyc-open-data":"yellow-taxi"
+    Given I "create" the following documents:
+      | _id       | body                             |
+      | "antoine" | { "name": "Antoine Ducuroy" }    |
+      | "karina"  | { "name": "Karina Tsimashenka" } |
+    Given I refresh the collection
+    Given I run the command "collection:export" with:
+      | arg | nyc-open-data |  |
+      | arg | yellow-taxi   |  |
+    Given I successfully call the route "collection":"truncate" with args:
+      | index      | "nyc-open-data" |
+      | collection | "yellow-taxi"   |
+    # migrate
+    When I run the command "collection:migrate" with args:
+      | "./features/fixtures/migration.js" |
+      | "./nyc-open-data/yellow-taxi"      |
+    Then The document "antoine" content match:
+      | firstName | "Antoine" |
+      | lastName  | "Ducuroy" |
+    Then The document "karina" content match:
+      | firstName | "Karina"      |
+      | lastName  | "Tsimashenka" |

features/fixtures/mappings.js

@@ -5,6 +5,8 @@ module.exports = {
       properties: {
         name: { type: "keyword" },
         city: { type: "keyword" },
+        firstName: { type: "keyword" },
+        lastName: { type: "keyword" }
       },
     },
   },

features/fixtures/migration.js

@@ -0,0 +1,11 @@
+function migrate(document) {
+  const [firstName, lastName] = document.body.name.split(' ');
+
+  return {
+    _id: document._id,
+    body: {
+      firstName,
+      lastName,
+    }
+  };
+}

src/commands/app/scaffold.ts

@@ -14,7 +14,8 @@ export default class AppScaffold extends Kommand {
     help: flags.help(),
     flavor: flags.string({
       default: "generic",
-      description: 'Template flavor ("generic", "iot-platform")',
+      description: `Template flavor ("generic", "iot-platform", "iot-console", "iot-platform").
+    Those can be found here: https://github.com/kuzzleio/project-templates`,
     }),
   };
 

src/commands/collection/migrate.ts

@@ -0,0 +1,70 @@
+import path from "path";
+import fs from "fs";
+
+import { flags } from "@oclif/command";
+
+import { Kommand } from "../../common";
+import { kuzzleFlags } from "../../support/kuzzle";
+import { restoreCollectionData } from "../../support/restore-collection";
+import { JSONObject } from "kuzzle-sdk";
+
+export default class CollectionMigrate extends Kommand {
+  static keepAuth = true;
+
+  static description = "Migrate a collection";
+
+  static flags = {
+    help: flags.help({}),
+    "batch-size": flags.string({
+      description: "Maximum batch size (see limits.documentsWriteCount config)",
+      default: "200",
+    }),
+    index: flags.string({
+      description: "If set, override the index destination name",
+    }),
+    collection: flags.string({
+      description: "If set, override the collection destination name",
+    }),
+    ...kuzzleFlags,
+    protocol: flags.string({
+      description: "Kuzzle protocol (http or websocket)",
+      default: "ws",
+    }),
+  };
+
+  static args = [
+    { name: "script", description: "Migration script path", required: true },
+    { name: "path", description: "Collection dump path", required: true },
+  ];
+
+  async runSafe() {
+    this.logInfo(`Start migration documents with ${this.args.script} from ${this.args.path}`);
+
+    const migrateDocument = this.getMigrationFunction();
+
+    const { index, collection, total } = await restoreCollectionData(
+      this.sdk,
+      this.log.bind(this),
+      Number(this.flags["batch-size"]),
+      path.join(this.args.path, "documents.jsonl"),
+      this.flags.index,
+      this.flags.collection,
+      migrateDocument
+    );
+
+    this.logOk(
+      `Successfully migrated ${total} documents from "${this.args.path}" in "${index}:${collection}"`
+    );
+  }
+
+  private getMigrationFunction(): (document: JSONObject) => JSONObject {
+    const migrationScript = fs.readFileSync(this.args.script, "utf8");
+
+    let migrationFunction;
+    {
+      migrationFunction = eval(`var f = ${migrationScript}; f`);
+    }
+
+    return migrationFunction;
+  }
+}

src/support/dump-collection.ts

@@ -83,14 +83,14 @@ abstract class AbstractDumper {
    * @returns void
    */
   // eslint-disable-next-line @typescript-eslint/no-empty-function
-  public async setup() {}
+  public async setup() { }
 
   /**
    * One-shot call before iterating over the data. Can be
    * used to write the header of the dumped output.
    */
   // eslint-disable-next-line @typescript-eslint/no-empty-function
-  public async writeHeader() {}
+  public async writeHeader() { }
 
   /**
    * You can put here the logic to write into the dump.
@@ -111,7 +111,7 @@ abstract class AbstractDumper {
   abstract onResult(document: { _id: string; _source: any }): Promise<void>;
 
   // eslint-disable-next-line @typescript-eslint/no-empty-function
-  public async tearDown() {}
+  public async tearDown() { }
 
   /**
    * The loop that iterates over the documents of the collection and
@@ -254,6 +254,11 @@ class CSVDumper extends AbstractDumper {
         return;
       }
       this.fields = Object.keys(flattenObject(mappings.properties));
+      for (const field of this.fields) {
+        if (field.endsWith(".type")) {
+          this.fields.splice(this.fields.indexOf(field), 1);
+        }
+      }
     } else {
       // Delete '_id' from the selected fields, since IDs are
       // _always_ exported.

src/support/restore-collection.ts

@@ -1,6 +1,8 @@
-import chalk from "chalk";
 import fs from "fs";
+
+import chalk from "chalk";
 import ndjson from "ndjson";
+import { JSONObject } from "kuzzle-sdk";
 
 function handleError(log: any, dumpFile: string, error: any) {
   if (error.status === 206) {
@@ -34,7 +36,8 @@ export async function restoreCollectionData(
   batchSize: number,
   dumpFile: string,
   index?: string,
-  collection?: string
+  collection?: string,
+  migrateDocument?: (document: JSONObject) => JSONObject
 ) {
   const mWriteRequest = {
     controller: "bulk",
@@ -57,7 +60,14 @@ export async function restoreCollectionData(
       .pipe(ndjson.parse())
       .on("data", (obj: any) => {
         if (headerSkipped) {
-          documents.push(obj);
+          const document = migrateDocument ? migrateDocument(obj) : obj;
+          if (!document._id) {
+            throw new Error(`Document does not have an "_id" property: ${JSON.stringify(document)}`);
+          }
+          if (!document.body) {
+            throw new Error(`Document does not have an "body" property: ${JSON.stringify(document)}`);
+          }
+          documents.push(document);
 
           if (documents.length === batchSize) {
             mWriteRequest.body.documents = documents;