More Typechecker improvements (#173)

README.md

@@ -728,8 +728,6 @@ foo(1, 2, 3); // Error in TS
 
 ---
 
-For performance reasons, we recommend a separate typechecker pass (in parallel with the main build), with `--check-types` enabled, `--output-language` set to `none`, and TypeScript type definitions (such as those found at [DefinitelyTyped](http://definitelytyped.org)) specified using the `--prepend` option.
-
 NilScript tries to convert TypeScript error messages back into NilScript syntax.  Please report any confusing error messages.
 
 ---
@@ -843,8 +841,7 @@ squeeze | Object  | Map of squeezed identifiers to original identifiers.  See [S
 
 The `before-compile` key specifies a callback which is called prior to the compiler's NilScript->js stage.  This allows you to preprocess files.  The callback must return a Promise. Once the promise is resolved, a file's content must be valid NilScript or JavaScript.
 
-The `after-compile` key specifies a callback which is called each time the compiler generates JavaScript code for a file.  This allows you to run the generated JavaScript through a linter (such as [ESLint](http://eslint.org)), or allows further transformations via [Babel](https://babeljs.io).
-The callback must return a Promise. When this callback is invoked, a file's content will be valid JavaScript.
+The `after-compile` key specifies a callback which is called each time the compiler generates JavaScript code for a file.  This allows you to run the generated JavaScript through a linter (such as [ESLint](http://eslint.org)), or allows further transformations via [Babel](https://babeljs.io). The callback must return a Promise. When this callback is invoked, a file's content will be valid JavaScript.
 
 
 ```javascript
@@ -906,6 +903,24 @@ Note: `options.state` and `result.state` are private objects and the format/cont
 
 NilScript 2.x also adds the `symbolicate` function as API.  This converts an internal NilScript identifier such as `N$_f_stringWithString_` to a human-readable string (`"stringWithString:"`).  See [Squeezing and Symbolication](#squeeze) below.
 
+--
+
+To improve type checker performance, NilScript 3.x adds a `tuneTypecheckerPerformance` API:
+
+`nilscript.tuneTypecheckerPerformance(includeInCompileResults, workerCount)`
+
+Key                       | Type     | Default |
+------------------------- | -------- | ------- |
+includeInCompileResults   | Boolean  | `true`  |
+workerCount               | Number   | `4`     |
+
+When `includeInCompileResults` is `true`, Each call to `Compiler#compile` will wait for its associated type checker to finish. Type checker warnings are then merged with `results.warnings`.
+
+When `includeInCompileResults` is `false`, `Compiler#compile` will start the type checker but not wait for it to finish. Warnings are accessed via the `Promise` returned from `Compiler#collectTypecheckerWarnings`. In complex projects with several `Compiler` objects, this option can result in faster compile times.
+
+`workerCount` sets the number of node `worker_threads` used to run TypeScript compilers.
+
+
 ---
 ## <a name="compiling-projects"></a>Compiling Projects
 
@@ -960,15 +975,6 @@ function doWebAppCompile(callback) {
 5. Both `core.js` and `webapp.js` are included (in that order) in various HTML files via `<script>` elements.
 6. The NilScript runtime (`runtime.js`) is also included in various HTML files.  You can obtain its location via the `getRuntimePath` API.
 
---
-
-We've found it best to run a separate typecheck pass in parallel with the `core.js`/`webapp.js` build (via a separate `node` process).  This allows one CPU to be dedicated to typechecking while the other performs transpiling.  The typecheck pass uses the following options:
-
-* All `.js` and `.ns` files (From steps #1 and #3) are passed as `INPUT_FILES` (or `options.files`).
-* Several `.d.ts` definitions (for jQuery, underscore, etc.) are specified with the `--defs` option (or `options.defs`).
-* `--output-language` is set to `none`.
-* `--check-types` is enabled 
-
 ---
 ## <a name="squeeze"></a>Squeezing and Symbolication
 

lib/api.js

@@ -1,5 +1,6 @@
 
 import { Compiler as CompilerImpl } from "../src/Compiler.js";
+import { tuneTypecheckerPerformance } from "../src/typechecker/Typechecker.js";
 import { symbolicate } from "../src/model/NSSymbolTyper.js";
 import { Utils } from "../src/Utils.js";
 
@@ -23,20 +24,30 @@ async compile(options)
 }
 
 
+async collectTypecheckerWarnings()
+{
+    return this._impl.collectTypecheckerWarnings();
+}
+
+
 }
 
 
 export default {
-    Compiler: Compiler,
+    Compiler,
 
     compile: async function(options) {
         return (new Compiler()).compile(options);
     },
-
+    
     getRuntimePath: function() {
         return Utils.getProjectPath("lib/runtime.js");
     },
 
+    tuneTypecheckerPerformance(includeInCompileResults, workerCount) {
+        return tuneTypecheckerPerformance(includeInCompileResults, workerCount);
+    },
+
     symbolicate: function(symbol) {
         return symbolicate(symbol);
     }

src/Compiler.js

@@ -55,7 +55,7 @@ const sPublicOptions = [
     // Typechecker options
     "check-types",               // Boolean, enable type checker
     "defs",                      // String or Object, additional typechecker defs
-    "typescript-lib",            // String, specify alternate lib.d.ts file
+    "typescript-lib",            // String, specify alternate lib.d.ts file(s)
     "no-implicit-any",           // Boolean, disallow implicit any
     "no-implicit-returns",       // Boolean, disallow implicit returns
     "no-unreachable-code",       // Boolean, inverts tsc's "--allowUnreachableCode" option
@@ -70,7 +70,7 @@ const sPublicOptions = [
 
     // Private / Development
     "dev-dump-tmp",              // Boolean, dump debug info to /tmp
-    "dev-print-log",             // Boolean, print log to stdout 
+    "dev-print-log",             // Boolean, print log to stdout
     "allow-private-options",     // Boolean, allow use of sPrivateOptions (see below)
 ];
 let sPublicOptionsMap = null;
@@ -96,6 +96,7 @@ constructor()
     this._model   = null;   
     this._parents = null;
     this._checker = null;
+    this._checkerPromise = null;
 }
 
 
@@ -232,7 +233,6 @@ async _parseFiles(files, options)
                 nsFile.ast = Parser.parse(nsFile.contents, { loc: true, sourceType: sourceType });
 
                 nsFile.needsGenerate();
-                nsFile.needsTypecheck();
 
             } catch (inError) {
                 let message = inError.description || inError.toString();
@@ -332,23 +332,23 @@ async _generateJavaScript(files, model, options)
 
     await Promise.all(_.map(files, async nsFile => {
         try {
-            if (!nsFile.generatorLines) {
+            if (!nsFile.generatedLines) {
                 Log(`Generating ${nsFile.path}`);
 
-                let generator = new Generator(nsFile, model, false, options);
+                let generator = new Generator(nsFile, model, options);
                 let result    = generator.generate();
 
-                nsFile.generatorLines    = result.lines;
-                nsFile.generatorWarnings = result.warnings || [ ];
-            }
+                nsFile.generatedLines    = result.lines;
+                nsFile.generatedWarnings = result.warnings || [ ];
 
-            if (afterCompileCallback) {
-                let callbackFile = new NSCompileCallbackFile(nsFile.path, nsFile.generatorLines, nsFile.generatorWarnings);
+                if (afterCompileCallback) {
+                    let callbackFile = new NSCompileCallbackFile(nsFile.path, nsFile.generatedLines, nsFile.generatedWarnings);
 
-                await afterCompileCallback(callbackFile);
+                    await afterCompileCallback(callbackFile);
 
-                nsFile.generatorLines    = callbackFile._lines;
-                nsFile.generatorWarnings = callbackFile._warnings;
+                    nsFile.generatedLines    = callbackFile._lines;
+                    nsFile.generatedWarnings = callbackFile._warnings;
+                }
             }
 
         } catch (err) {
@@ -363,31 +363,6 @@ async _generateJavaScript(files, model, options)
 }
 
 
-async _runTypechecker(typechecker, defs, files, model, options)
-{
-    let err = null;
-
-    await Promise.all(_.map(files, async nsFile => {
-        if (!nsFile.typecheckerCode) {
-            try {
-                let generator = new Generator(nsFile, model, true, options);
-                let result = generator.generate();
-
-                nsFile.typecheckerCode = result.lines.join("\n");
-
-            } catch (e) {
-                nsFile.needsTypecheck();
-                nsFile.error = e;
-            }
-        }
-    }));
-
-    this._throwErrorInFiles(files);
-
-    return typechecker.check(model, defs, files);
-}
-
-
 async _finish(files, options)
 {
     function getLines(arrayOrString) {
@@ -437,7 +412,7 @@ async _finish(files, options)
 
         linesArray.push(prependLines);
         _.each(files, nsFile => {
-            linesArray.push(nsFile.generatorLines);
+            linesArray.push(nsFile.generatedLines);
         });
         linesArray.push(appendLines);
 
@@ -459,6 +434,12 @@ uses(compiler)
 }
 
 
+async collectTypecheckerWarnings()
+{
+    return this._checker?.collectWarnings();
+}
+
+
 async compile(options)
 {
     let previousFiles   = this._files;
@@ -529,16 +510,32 @@ async compile(options)
         });
 
         this._checker = null;
+        this._checkerPromise = null;
+    }
+
+    if (optionsCheckTypes && !this._checker) {
+        this._checker = new Typechecker(options);
     }
 
     let model = new NSModel();
+
     if (this._parents) {
+        let parentCheckers = [ ];
+
         _.each(this._parents, parent => {
             if (parent._model) {
                 model.loadState(parent._model.saveState());
             }
+
+            if (parent._checker) {
+                parentCheckers.push(parent._checker);
+            }
         });
 
+        if (this._checker) {
+            this._checker.parents = parentCheckers;
+        }
+
     } else if (optionsState) {
         model.loadState(optionsState);
     } 
@@ -554,10 +551,6 @@ async compile(options)
     this._options = options;
     this._defs    = defs;
 
-    if (optionsCheckTypes && !this._checker) {
-        this._checker = new Typechecker(options);
-    }
-
     let outputCode          = null;
     let outputSourceMap     = null;
     let outputFunctionMap   = null;
@@ -591,10 +584,7 @@ async compile(options)
                 Log("Model has global changes, all files need generate");
             }
 
-            _.each(files, nsFile => {
-                nsFile.needsGenerate();
-                nsFile.needsTypecheck();
-            });
+            _.each(files, nsFile => nsFile.needsGenerate());
 
         } else {
             if (options["warn-unknown-selectors"]) {
@@ -616,16 +606,20 @@ async compile(options)
         // If we get here, our current model is valid.  Save it for next time
         this._model = model;
 
+        // Run typechecker
+        if (optionsCheckTypes) {
+            this._checker.check(model, defs, files);
+
+            if (Typechecker.includeInCompileResults) {
+                typecheckerWarnings = await this._checker.collectWarnings();
+            }
+        }
+
         // Run generator
         if (optionsOutputLanguage != "none") {
             await this._generateJavaScript(files, model, options);
         }
 
-        // Run typechecker
-        if (optionsCheckTypes) {
-            typecheckerWarnings = await this._runTypechecker(this._checker, defs, files, model, options);
-        }
-
         // Concatenate and map output
         if (optionsOutputLanguage != "none") {
             let results = await this._finish(files, finishOptions);
@@ -654,7 +648,7 @@ async compile(options)
     });
 
     let warnings = _.map(files, nsFile => [
-        nsFile.generatorWarnings,
+        nsFile.generatedWarnings,
     ]);
 
     warnings.push(typecheckerWarnings);

src/Generator.js

@@ -31,7 +31,7 @@ const LanguageNone        = "none";
 export class Generator {
 
 
-constructor(nsFile, model, forTypechecker, options)
+constructor(nsFile, model, options)
 {
     this._file     = nsFile;
     this._model    = model;
@@ -50,16 +50,11 @@ constructor(nsFile, model, forTypechecker, options)
         this._language = LanguageEcmascript5;
     }
 
-    if (forTypechecker || (this._language == LanguageTypechecker)) {
-        this._language = LanguageTypechecker;
-        forTypechecker = true;
-    }
-
     _.each(model.enums, nsEnum => {
         let enumNameSymbol = (nsEnum.name && !nsEnum.anonymous) ? symbolTyper.getSymbolForEnumName(nsEnum.name) : null;
 
         _.each(nsEnum.values, (value, name) => {
-            if (enumNameSymbol && forTypechecker) {
+            if (enumNameSymbol && (this._language == LanguageTypechecker)) {
                 inlines[name] = enumNameSymbol + "." + name;
             } else {
                 inlines[name] = value;

src/Utils.js

@@ -128,23 +128,6 @@ function log()
 }
 
 
-function rmrf(dir)
-{
-    try {
-        _.each(fs.readdirSync(dir), file => {
-            file = dir + path.sep + file;
-            if (fs.statSync(file).isFile()) {
-                fs.unlinkSync(file)
-            } else {
-                rmrf(file);
-            }
-        });
-
-        fs.rmdirSync(dir);
-    } catch(e) { }
-}
-
-
 function mkdirAndWriteFile(file, contents)
 {
     fs.mkdirSync(path.dirname(file), { recursive: true });
@@ -173,6 +156,5 @@ export const Utils = {
     enableLog,
     log,
 
-    rmrf,
     mkdirAndWriteFile
 };