Feature/1.0.0/fork

Makefile

@@ -73,7 +73,7 @@ post-coverage:
 
 # The command the ci server runs
 ci:
-	make lint build-source-maps -i
+	make lint || exit 1
 	# if the tests fail then it will exit with an error
 	make coverage || exit 1
 	# show the coverage report

app/documents.js

@@ -1,3 +1,9 @@
+let exit = false;
+process.on('SIGINT', () => {
+  exit = true;
+  process.exit(2);
+});
+
 import faker from 'faker';
 import Chance from 'chance';
 import Base from './base';
@@ -6,13 +12,111 @@ import { set, get } from 'lodash';
 import to from 'to-js';
 
 ////
-/// @name Document
-/// @page api/document
+/// @name Documents
+/// @page api/documents
 ////
 
+/// @name Documents
+/// @description This class is used to generate all the documents for the passed models
+export default class Documents extends Base {
+  constructor(options = {}, documents = {}, globals = {}, inputs = {}) {
+    super(options);
+    this.options = to.extend({ count: 0 }, this.options);
+    this.documents = documents;
+    this.globals = globals;
+    this.inputs = inputs;
+    this.total = 0;
+  }
+
+  ///# @name build
+  ///# @description
+  ///# This takes an array of models and builds them
+  ///# @arg {array} models - Array of models
+  ///# @returns {array} of documents
+  ///# @async
+  build(models) {
+    models = to.clone(models);
+    const todo = models.map((model) => model.file);
+    const result = [];
+
+    function finished(dependencies) {
+      for (let file of dependencies) {
+        for (let model of models) {
+          if (model.file === file && !model.complete) {
+            return false;
+          }
+        }
+      }
+
+      return true;
+    }
+
+    this.on('run', () => {
+      // don't run again if the program should exit
+      if (exit) return;
+      for (let model of models) {
+        const index = todo.indexOf(model.file);
+        if (index > -1 && finished(model.data.dependencies)) {
+          todo.splice(index, 1);
+          const document = this.document(model);
+
+          if (!model.is_dependency) {
+            result.push(document);
+          }
+        }
+        if (model.complete && finished(model.dependants)) {
+          // if the model is no longer needed then remove it
+          delete this.documents[model.name];
+        }
+      }
+
+      if (!todo.length) {
+        this.emit('finished');
+      }
+    });
+
+    this.emit('run');
+
+    return new Promise((resolve, reject) => {
+      this.once('error', (err) => {
+        reject(err);
+      });
+      this.once('finished', () => {
+        Promise.all(result).then((res) => {
+          this.inputs = {};
+          resolve(res);
+        });
+      });
+    });
+  }
+
+  document(model) {
+    const document = new Document(this.options, this.documents, this.globals, this.inputs);
+    return document.build(model)
+      .then(() => {
+        let result;
+        if (!model.is_dependency) {
+          result = this.emit('data', this.documents[model.name], model);
+        }
+        // update the total documents number
+        this.total += this.documents[model.name].length;
+        model.complete = true;
+        this.emit('run');
+        return result;
+      })
+      .catch((err) => {
+        exit = true;
+        process.exit(2);
+        this.emit('error', err);
+      });
+  }
+}
+
+
+
 /// @name Document
 /// @description This is used to generate documents based off a model
-export default class Document extends Base {
+export class Document extends Base {
   constructor(options = {}, documents = {}, globals = {}, inputs = {}) {
     super(options);
     this.options = to.extend({ count: 0 }, this.options);
@@ -59,12 +163,19 @@ export default class Document extends Base {
     spinner.start();
     const delay = (duration) => new Promise((resolve) => setTimeout(resolve, duration));
     await pool(model.data.count, async (i) => { // loop over each model and execute in order of dependency
+      // don't anymore if the program should exit
+      if (exit) return;
+
+      update();
       // this allows the spinner to actually update the count and doesn't affect performance much
       const doc = await this.buildDocument(model, i);
       update();
       await delay(0);
       this.documents[model.name].push(doc);
-    }, this.options.spinners ? 75 : 1000);
+    }, this.options.spinners ? 75 : 1000)
+      .catch((err) => {
+        spinner.fail(err);
+      });
 
     this.runData(model.data.post_run, model);
 

app/index.js

@@ -1,10 +1,9 @@
 import Models from './models';
-import { map } from 'async-array-methods';
 import Output from './output/index';
 import Base from './base';
 import to from 'to-js';
 import { uniqueId } from 'lodash';
-import Document from './documents';
+import Documents from './documents';
 import { success } from 'log-symbols';
 
 /// @name Fakeit
@@ -53,27 +52,17 @@ export default class Fakeit extends Base {
     output.prepare();
 
     await model.registerModels(models);
-
-    const document = new Document(this.options, this.documents, this.globals, model.inputs);
-
-    let result = [];
-
-    for (let obj of to.flatten(model.models)) {
-      const value = await document.build(obj);
-      if (!obj.is_dependency) {
-        result.push(value);
-      }
-    }
-
-    result = await Promise.all(result);
-
     await output.preparing;
-    result = await map(result, (data) => output.output(data));
+
+    const documents = new Documents(this.options, this.documents, this.globals, model.inputs);
+    delete model.inputs;
+    let result = documents.build(model.models);
+    documents.on('data', (data) => output.output(data));
+    result = await result;
     await output.finalize();
     const time = this.timeEnd(label);
-    const total = to.reduce(document.documents, (prev, { value }) => prev + value.length, 0);
     if (this.options.verbose) {
-      console.log(`${success} Finished generating ${total} documents in ${time}`);
+      console.log(`${success} Finished generating ${documents.total} documents in ${time}`);
     }
 
     return result;

app/logger.js

@@ -6,14 +6,18 @@ symbols.warn = symbols.warning;
 symbols.ok = symbols.okay = symbols.success;
 import ora from 'ora';
 import formatSeconds from 'format-seconds';
+import Emitter from 'events-async';
+
 
 /// @name Logger
 /// @description
 /// This is the main logger for the application
-export default class Logger {
+export default class Logger extends Emitter {
   ///# @name constructor
   ///# @arg {object} options [{ log: true, verbose: false, timestamp: true }]
   constructor(options = {}) {
+    super();
+    this.setMaxListeners(50);
     this.options = to.extend({
       log: true,
       verbose: false,

app/models.js

@@ -30,8 +30,6 @@ export default class Models extends Base {
     this.registered_models = []; // holds the paths that have already been added
 
     this.prepared = false;
-
-    this.progress = this.spinner('Models');
   }
 
   ///# @name prepare
@@ -109,7 +107,7 @@ export default class Models extends Base {
   }
 
   async registerModels(models, dependency = false) {
-    this.progress.start();
+    this.progress = this.progress || this.spinner('Models').start();
     // if models weren't passed in then don't do anything
     if (!models) {
       return;
@@ -189,9 +187,12 @@ export default class Models extends Base {
 
     // update the models order
     this.models = resolveDependenciesOrder(this.models);
+    this.models = resolveDependants(this.models);
 
     if (this.models.length === this.registered_models.length) {
       this.progress.stop();
+      // unset progress so when it's run again it works correctly
+      // this.progress = null;
     }
     return this;
   }
@@ -216,7 +217,7 @@ export default class Models extends Base {
     parseModelSeed(model, this.options.seed);
 
     // add this models inputs to the main inputs object
-    this.inputs = to.extend(this.inputs || {}, await inputs);
+    this.inputs = Object.assign(this.inputs || {}, await inputs);
     await dependencies;
     return model;
   }
@@ -236,7 +237,7 @@ export default class Models extends Base {
     }
 
     // get list of files, flatten the array of files and filter files for valid input formats: yaml
-    const files = this.filterModelFiles(await utils.findFiles(model.data.dependencies));
+    const files = to.flatten(await utils.findFiles(model.data.dependencies));
 
     if (!files.length) {
       model.data.dependencies = [];
@@ -260,7 +261,7 @@ export async function parseModelInputs(model) {
     !model.data.inputs ||
     !model.data.inputs.length
   ) {
-    model.data.inputs = {};
+    model.data.inputs = [];
     return {};
   }
 
@@ -286,7 +287,6 @@ export async function parseModelInputs(model) {
     return file;
   });
 
-  model.data.inputs = inputs;
   return inputs;
 }
 
@@ -481,3 +481,24 @@ export function resolveDependenciesOrder(models = []) {
 
   return resolver.sort().map((file) => models[order[file]]);
 }
+
+
+/// @name resolveDependenciesOf
+/// @description Figures out which models use the model as a dependency
+/// @arg {array} models [[]] - The models to loop over
+/// @returns {array} - The models are returned with the `dependants`
+export function resolveDependants(models = []) {
+  return models.map((model) => {
+    model.dependants = models.reduce((prev, next) => {
+      if (
+        model.file !== next.file &&
+        next.data.dependencies.includes(model.file)
+      ) {
+        prev.push(next.file);
+      }
+      return prev;
+    }, []);
+
+    return model;
+  });
+}

package.json

@@ -46,20 +46,21 @@
   "dependencies": {
     "adm-zip": "^0.4.7",
     "async-array-methods": "^2.1.0",
-    "babel-core": "~6.20.0",
-    "babel-helpers": "~6.16.0",
-    "babel-runtime": "^6.18.0",
+    "babel-core": "~6.22.1",
+    "babel-helpers": "~6.22.0",
+    "babel-runtime": "~6.22.0",
     "chalk": "^1.1.3",
     "chance": "^1.0.4",
     "cli-table": "~0.3.1",
     "commander": "^2.9.0",
-    "couchbase-promises": "~2.2.0",
+    "couchbase-promises": "~3.0.1",
     "cson": "^4.0.0",
     "csv-parse": "^1.1.7",
     "csv-stringify": "^1.0.4",
     "dependency-resolver": "~2.0.1",
     "es6-promise-pool": "^2.4.4",
     "es6-promisify": "^5.0.0",
+    "events-async": "~1.0.0",
     "faker": "^3.1.0",
     "find-root": "~1.0.0",
     "format-seconds": "~0.3.1",
@@ -71,29 +72,29 @@
     "ora": "~0.4.1",
     "perfy": "~1.1.2",
     "request": "^2.78.0",
-    "set-cookie-parser": "^1.0.2",
+    "set-cookie-parser": "~2.0.0",
     "to-js": "^0.0.6",
     "update-notifier": "^1.0.2",
     "yamljs": "^0.2.8"
   },
   "devDependencies": {
     "ava": "^0.16.0",
     "ava-spec": "github:tjbenton/ava-spec",
-    "babel-cli": "^6.18.0",
+    "babel-cli": "~6.22.2",
     "babel-plugin-array-includes": "~2.0.3",
-    "babel-plugin-external-helpers": "~6.18.0",
-    "babel-plugin-transform-runtime": "~6.15.0",
-    "babel-preset-latest": "^6.16.0",
-    "babel-preset-stage-0": "^6.16.0",
+    "babel-plugin-external-helpers": "~6.22.0",
+    "babel-plugin-transform-runtime": "~6.22.0",
+    "babel-preset-latest": "~6.22.0",
+    "babel-preset-stage-0": "~6.22.0",
     "coveralls": "^2.11.15",
     "docs-core": "0.0.0-alpha-6.1",
     "docs-theme-default": "0.0.0-alpha-6.1",
     "get-stream": "~3.0.0",
-    "joi": "~10.0.5",
+    "joi": "~10.2.0",
     "jsondiffpatch": "~0.2.4",
     "lint-rules": "github:ma-shop/lint-rules",
     "nixt": "~0.5.0",
-    "nyc": "~9.0.1",
+    "nyc": "~10.1.2",
     "proxyquire": "~1.7.10",
     "test-console": "~1.0.0"
   },