[DO NOT REVIEW] Perf testing module-ified TypeScript compiler

.eslintrc.json

@@ -94,7 +94,8 @@
         "local/simple-indent": "error",
         "local/debug-assert": "error",
         "local/no-keywords": "error",
-        "local/one-namespace-per-file": "error",
+        // TOOD(jakebailey): delete; no longer relavent after modules.
+        // "local/one-namespace-per-file": "error",
 
         // eslint-plugin-import
         "import/no-extraneous-dependencies": ["error", { "optionalDependencies": false }],

Gulpfile.js

@@ -19,6 +19,8 @@ const cmdLineOptions = require("./scripts/build/options");
 const copyright = "CopyrightNotice.txt";
 const cleanTasks = [];
 
+const testRunner = "./built/local/testRunner.js";
+
 const buildScripts = () => buildProject("scripts");
 task("scripts", buildScripts);
 task("scripts").description = "Builds files in the 'scripts' folder.";
@@ -55,9 +57,7 @@ const diagnosticInformationMapTs = "src/compiler/diagnosticInformationMap.genera
 const diagnosticMessagesJson = "src/compiler/diagnosticMessages.json";
 const diagnosticMessagesGeneratedJson = "src/compiler/diagnosticMessages.generated.json";
 const generateDiagnostics = async () => {
-    if (needsUpdate(diagnosticMessagesJson, [diagnosticMessagesGeneratedJson, diagnosticInformationMapTs])) {
-        await exec(process.execPath, ["scripts/processDiagnosticMessages.js", diagnosticMessagesJson]);
-    }
+    await exec(process.execPath, ["scripts/processDiagnosticMessages.js", diagnosticMessagesJson]);
 };
 task("generate-diagnostics", series(buildScripts, generateDiagnostics));
 task("generate-diagnostics").description = "Generates a diagnostic file in TypeScript based on an input JSON file";
@@ -87,20 +87,114 @@ const localizationTargets = ["cs", "de", "es", "fr", "it", "ja", "ko", "pl", "pt
     .concat(generatedLCGFile);
 
 const localize = async () => {
-    if (needsUpdate(diagnosticMessagesGeneratedJson, generatedLCGFile)) {
-        return exec(process.execPath, ["scripts/generateLocalizedDiagnosticMessages.js", "src/loc/lcl", "built/local", diagnosticMessagesGeneratedJson], { ignoreExitCode: true });
+    return exec(process.execPath, ["scripts/generateLocalizedDiagnosticMessages.js", "src/loc/lcl", "built/local", diagnosticMessagesGeneratedJson], { ignoreExitCode: true });
+};
+
+/**
+ * @param {string} entrypoint
+ * @param {string} outfile
+ */
+async function esbuild(entrypoint, outfile) {
+    await exec("node_modules/esbuild/bin/esbuild", [
+        entrypoint,
+        "--bundle",
+        `--outfile=${outfile}`,
+        "--platform=node",
+        "--target=node12",
+        "--format=esm",
+        "--sourcemap",
+        "--external:./node_modules/*",
+        "--conditions=require",
+        // "--supported:const-and-let=false", // Unfortunately, no: https://github.com/evanw/esbuild/issues/297
+        "--supported:object-rest-spread=false", // See: https://github.com/evanw/esbuild/releases/tag/v0.14.46
+    ]);
+}
+
+const buildSrc = () => buildProject("src");
+
+/** @type {(infile: string, outfile: string) => (done: () => void) => void} */
+const writeHackyCJSShim = (infile, outfile) => {
+    return (done) => {
+        const inRelativeToOut = infile = path.relative(path.dirname(outfile), infile);
+        fs.writeFileSync(outfile, `module.exports = require("./${inRelativeToOut}")`);
+        done();
+    };
+};
+
+const writePackageJsonWithModuleType = (done) => {
+    fs.writeFileSync("./built/local/package.json", JSON.stringify({ name: "local", private: true, type: "module" }));
+    done();
+};
+
+/** @type {(infile: string, outfile: string) => (done: () => void) => void} */
+const writeHackyMJSShim = (infile, outfile) => {
+    const writeHackyMJSShim = (done) => {
+        const inRelativeToOut = infile = path.relative(path.dirname(outfile), infile);
+        fs.writeFileSync(outfile, `import "./${inRelativeToOut}"`);
+        done();
+    };
+    return writeHackyMJSShim;
+};
+
+const preBundleFromSrc = parallel(generateLibs, series(buildScripts, generateDiagnostics, localize));
+const preBuildSrc = preBundleFromSrc;
+const preBundleFromEmit = series(preBundleFromSrc, buildSrc);
+
+const bundleTscFromEmit = () => esbuild("./built/local/tsc/tsc.js", "./built/local/tsc.js");
+const bundleTypescriptFromEmit = () => esbuild("./built/local/typescript/typescript.js", "./built/local/typescript.js");
+const bundleServerFromEmit = () => esbuild("./built/local/tsserver/server.js", "./built/local/tsserver.js");
+const bundleServerLibraryFromEmit = () => esbuild("./built/local/tsserverlibrary/tsserverlibrary.js", "./built/local/tsserverlibrary.js");
+const bundleTestsFromEmit = () => esbuild("./built/local/testRunner/_namespaces/Harness.js", testRunner);
+
+const bundleTscFromSrc = () => esbuild("./src/tsc/tsc.ts", "./built/local/tsc.js");
+const bundleTypescriptFromSrc = () => esbuild("./src/typescript/typescript.ts", "./built/local/typescript.js");
+const bundleServerFromSrc = () => esbuild("./src/tsserver/server.ts", "./built/local/tsserver.js");
+const bundleServerLibraryFromSrc = () => esbuild("./src/tsserverlibrary/tsserverlibrary.ts", "./built/local/tsserverlibrary.js");
+const bundleTestsFromSrc = () => esbuild("./src/testRunner/_namespaces/Harness.ts", testRunner);
+
+const bundleAllFromSrc = series([
+    bundleTscFromSrc,
+    bundleTypescriptFromSrc,
+    bundleServerFromSrc,
+    bundleServerLibraryFromSrc,
+    bundleTestsFromSrc,
+]);
+task("bundle-src", series(preBundleFromSrc, bundleAllFromSrc));
+
+
+task("buildSrc", preBundleFromEmit);
+
+const apiExtractor = async () => {
+    async function runApiExtractor(configPath) {
+        await exec(process.execPath, [
+            "node_modules/@microsoft/api-extractor/bin/api-extractor",
+            "run",
+            "--local",
+            "--config",
+            configPath,
+        ]);
     }
+
+    // TODO(jakebailey): prepend copyright notice, replace const enums with regular enums
+    await runApiExtractor("./src/typescript/api-extractor.json");
+    await runApiExtractor("./src/tsserverlibrary/api-extractor.json");
 };
 
+task("api-extractor", series(buildSrc, apiExtractor));
+
 const buildDebugTools = () => buildProject("src/debug");
 const cleanDebugTools = () => cleanProject("src/debug");
 cleanTasks.push(cleanDebugTools);
 
 // Pre-build steps when targeting the LKG compiler
-const lkgPreBuild = parallel(generateLibs, series(buildScripts, generateDiagnostics, buildDebugTools));
+const lkgPreBuild = parallel(generateLibs, series(buildScripts, generateDiagnostics /** , buildDebugTools */));
 
 const buildTsc = () => buildProject("src/tsc");
-task("tsc", series(lkgPreBuild, buildTsc));
+
+// task("tsc", series(preBundleFromSrc, bundleTscFromSrc)); // esbuild on ./src
+task("tsc", series(preBuildSrc, buildTsc, writePackageJsonWithModuleType, bundleTscFromEmit)); // esbuild on emitted ./built/local
+// task("tsc", series(preBuildSrc, buildSrc, writeHackyCJSShim("./built/local/tsc/tsc.js", "./built/local/tsc.js"))); // CJS
+// task("tsc", series(preBuildSrc, buildTsc, writePackageJsonWithModuleType, writeHackyMJSShim("./built/local/tsc/tsc.js", "./built/local/tsc.js"))); // MJS
 task("tsc").description = "Builds the command-line compiler";
 
 const cleanTsc = () => cleanProject("src/tsc");
@@ -119,6 +213,12 @@ const localPreBuild = parallel(generateLibs, series(buildScripts, generateDiagno
 const preBuild = cmdLineOptions.lkg ? lkgPreBuild : localPreBuild;
 
 const buildServices = (() => {
+    // TODO(jakebailey): fix this for modules
+    return cb => {
+        console.log("!!!TODO!!! buildServices");
+        cb();
+    };
+
     // build typescriptServices.out.js
     const buildTypescriptServicesOut = () => buildProject("src/typescriptServices/tsconfig.json", cmdLineOptions);
 
@@ -154,20 +254,20 @@ const buildServices = (() => {
         .pipe(rename("typescript.d.ts"))
         .pipe(dest("built/local"));
 
-    // create typescript_standalone.d.ts
-    const createTypescriptStandaloneDts = () => src("built/local/typescriptServices.d.ts")
-        .pipe(newer("built/local/typescript_standalone.d.ts"))
-        .pipe(transform(content => content.replace(/declare (namespace|module) ts/g, 'declare module "typescript"')))
-        .pipe(rename("typescript_standalone.d.ts"))
-        .pipe(dest("built/local"));
+    // // create typescript_standalone.d.ts
+    // const createTypescriptStandaloneDts = () => src("built/local/typescriptServices.d.ts")
+    //     .pipe(newer("built/local/typescript_standalone.d.ts"))
+    //     .pipe(transform(content => content.replace(/declare (namespace|module) ts/g, 'declare module "typescript"')))
+    //     .pipe(rename("typescript_standalone.d.ts"))
+    //     .pipe(dest("built/local"));
 
     return series(
         buildTypescriptServicesOut,
         createTypescriptServicesJs,
         createTypescriptServicesDts,
         createTypescriptJs,
         createTypescriptDts,
-        createTypescriptStandaloneDts,
+        // createTypescriptStandaloneDts,
     );
 })();
 task("services", series(preBuild, buildServices));
@@ -211,10 +311,20 @@ task("watch-services").flags = {
 const buildDynamicImportCompat = () => buildProject("src/dynamicImportCompat", cmdLineOptions);
 task("dynamicImportCompat", buildDynamicImportCompat);
 
+
+const buildTypingsInstaller = () => buildProject("src/typingsInstaller");
+const cleanTypingsInstaller = () => cleanProject("src/typingsInstaller");
+cleanTasks.push(cleanTypingsInstaller);
+
+
 const buildServerMain = () => buildProject("src/tsserver", cmdLineOptions);
 const buildServer = series(buildDynamicImportCompat, buildServerMain);
 buildServer.displayName = "buildServer";
-task("tsserver", series(preBuild, buildServer));
+
+// task("tsserver", series(preBundleFromSrc, bundleServerFromSrc)); // esbuild on ./src
+task("tsserver", series(preBuildSrc, buildTypingsInstaller, buildServer, writePackageJsonWithModuleType, bundleServerFromEmit)); // esbuild on emitted ./built/local
+// task("tsserver", series(preBuildSrc, buildSrc, writeHackyCJSShim("./built/local/tsserver/server.js", "./built/local/tsserver.js"))); // CJS
+// task("tsserver", series(preBuildSrc, buildTypingsInstaller, buildServerMain, writePackageJsonWithModuleType, writeHackyMJSShim("./built/local/tsserver/server.js", "./built/local/tsserver.js"))); // CJS
 task("tsserver").description = "Builds the language server";
 task("tsserver").flags = {
     "   --built": "Compile using the built version of the compiler."
@@ -252,6 +362,12 @@ task("watch-min").flags = {
 };
 
 const buildLssl = (() => {
+    // TODO(jakebailey): fix this for modules
+    return cb => {
+        console.log("!!!TODO!!! buildLssl");
+        cb();
+    };
+
     // build tsserverlibrary.out.js
     const buildServerLibraryOut = () => buildProject("src/tsserverlibrary/tsconfig.json", cmdLineOptions);
 
@@ -377,10 +493,6 @@ const buildCancellationToken = () => buildProject("src/cancellationToken");
 const cleanCancellationToken = () => cleanProject("src/cancellationToken");
 cleanTasks.push(cleanCancellationToken);
 
-const buildTypingsInstaller = () => buildProject("src/typingsInstaller");
-const cleanTypingsInstaller = () => cleanProject("src/typingsInstaller");
-cleanTasks.push(cleanTypingsInstaller);
-
 const buildWatchGuard = () => buildProject("src/watchGuard");
 const cleanWatchGuard = () => cleanProject("src/watchGuard");
 cleanTasks.push(cleanWatchGuard);
@@ -424,8 +536,8 @@ task("watch-local").flags = {
 const preTest = parallel(buildTsc, buildTests, buildServices, buildLssl);
 preTest.displayName = "preTest";
 
-const runTests = () => runConsoleTests("built/local/run.js", "mocha-fivemat-progress-reporter", /*runInParallel*/ false, /*watchMode*/ false);
-task("runtests", series(preBuild, preTest, runTests));
+const runTests = () => runConsoleTests(testRunner, "mocha-fivemat-progress-reporter", /*runInParallel*/ false, /*watchMode*/ false);
+task("runtests", series(/*preBuild, preTest,*/ preBundleFromSrc, bundleTestsFromSrc, runTests)); // TODO(jakebailey): fix this for modules
 task("runtests").description = "Runs the tests using the built run.js file.";
 task("runtests").flags = {
     "-t --tests=<regex>": "Pattern for tests to run.",
@@ -443,8 +555,8 @@ task("runtests").flags = {
     "   --shardId": "1-based ID of this shard (default: 1)",
 };
 
-const runTestsParallel = () => runConsoleTests("built/local/run.js", "min", /*runInParallel*/ cmdLineOptions.workers > 1, /*watchMode*/ false);
-task("runtests-parallel", series(preBuild, preTest, runTestsParallel));
+const runTestsParallel = () => runConsoleTests(testRunner, "min", /*runInParallel*/ cmdLineOptions.workers > 1, /*watchMode*/ false);
+task("runtests-parallel", series(/*preBuild, preTest,*/ preBundleFromSrc, bundleTestsFromSrc, runTestsParallel)); // TODO(jakebailey): fix this for modules
 task("runtests-parallel").description = "Runs all the tests in parallel using the built run.js file.";
 task("runtests-parallel").flags = {
     "   --light": "Run tests in light mode (fewer verifications, but tests run faster).",
@@ -592,10 +704,10 @@ task("publish-nightly").description = "Runs `npm publish --tag next` to create a
 // write some kind of trigger file that indicates build completion that we could listen for instead.
 const watchRuntests = () => watch(["built/local/*.js", "tests/cases/**/*.ts", "tests/cases/**/tsconfig.json"], { delay: 5000 }, async () => {
     if (cmdLineOptions.tests || cmdLineOptions.failed) {
-        await runConsoleTests("built/local/run.js", "mocha-fivemat-progress-reporter", /*runInParallel*/ false, /*watchMode*/ true);
+        await runConsoleTests(testRunner, "mocha-fivemat-progress-reporter", /*runInParallel*/ false, /*watchMode*/ true);
     }
     else {
-        await runConsoleTests("built/local/run.js", "min", /*runInParallel*/ true, /*watchMode*/ true);
+        await runConsoleTests(testRunner, "min", /*runInParallel*/ true, /*watchMode*/ true);
     }
 });
 task("watch", series(preBuild, preTest, parallel(watchLib, watchDiagnostics, watchServices, watchLssl, watchTests, watchRuntests)));