Add initial test262 support to test optional chaining

For now, we only run two subdirectories and ignore various tests (sloppy mode,
tests expecting syntax error, and TCO tests). This reveals a few issues with the
current implementation, which will be addressed later. Since it's not a passing
build yet, I'll leave it out of CI for now.

Also drop the Travis build for Node 8 since one of the new dependencies requires
Node 10. I won't officially drop Node 8 support just yet, but will soon, and
it's at end-of-life now.

.eslintrc.js

@@ -20,6 +20,7 @@ module.exports = {
           "**/example-runner/**",
           "**/generator/**",
           "**/test/**",
+          "**/test262/**",
           "**/script/**",
         ],
         optionalDependencies: false,

.gitignore

@@ -4,6 +4,7 @@ dist
 dist-self-build
 dist-types
 example-runner/example-repos
+test262/test262-checkout
 integrations/gulp-plugin/dist
 .nyc_output
 coverage.lcov

.travis.yml

@@ -32,10 +32,6 @@ matrix:
       script:
         - yarn build
         - yarn test-only
-    - node_js: '8'
-      script:
-        - yarn build
-        - yarn test-only
   # Exclude the default build; we only want to run explicitly-included builds.
   exclude:
     node_js: '12'

package.json

@@ -14,7 +14,7 @@
   "scripts": {
     "build": "sucrase-node script/build.ts",
     "fast-build": "sucrase-node script/build.ts --fast",
-    "clean": "rm -rf ./build ./dist ./dist-self-build ./dist-types ./example-runner/example-repos",
+    "clean": "rm -rf ./build ./dist ./dist-self-build ./dist-types ./example-runner/example-repos ./test262/test262-checkout",
     "generate": "sucrase-node generator/generate.ts",
     "benchmark": "sucrase-node benchmark/benchmark.ts",
     "microbenchmark": "sucrase-node benchmark/microbenchmark.ts",
@@ -28,6 +28,7 @@
     "run-examples": "sucrase-node example-runner/example-runner.ts",
     "test": "yarn lint && yarn test-only",
     "test-only": "mocha './test/**/*.ts'",
+    "test262": "sucrase-node test262/run-test262.ts",
     "test-with-coverage": "nyc mocha './test/**/*.ts'",
     "report-coverage": "nyc report --reporter=text-lcov > coverage.lcov && codecov"
   },
@@ -58,6 +59,7 @@
     "@types/node": "^12.12.7",
     "@types/yargs-parser": "^13.1.0",
     "@typescript-eslint/parser": "^2.6.1",
+    "chalk": "2.4.1",
     "codecov": "^3.6.1",
     "eslint": "^6.6.0",
     "eslint-config-airbnb-base": "^14.0.0",
@@ -69,6 +71,7 @@
     "nyc": "^14.1.1",
     "prettier": "^1.19.1",
     "sucrase": "^3.11.0",
+    "test262-harness": "^6.5.0",
     "ts-interface-builder": "^0.2.1",
     "tslint": "^5.20.1",
     "typescript": "^3.7.2",

script/lint.ts

@@ -29,7 +29,7 @@ async function checkSucrase(): Promise<void> {
     run(`${TSC} --project . --noEmit`),
     run(`${TSLINT} --project .`),
     run(
-      `${ESLINT} ${["benchmark", "example-runner", "generator", "script", "src", "test"]
+      `${ESLINT} ${["benchmark", "example-runner", "generator", "script", "src", "test", "test262"]
         .map((dir) => `'${dir}/**/*.ts'`)
         .join(" ")}`,
     ),

test262/run-test262.ts

@@ -0,0 +1,71 @@
+#!./node_modules/.bin/sucrase-node
+/* eslint-disable no-console */
+import chalk from "chalk";
+import {exec} from "mz/child_process";
+import {exists} from "mz/fs";
+
+import run from "../script/run";
+
+const TEST262_HARNESS = "./node_modules/.bin/test262-harness";
+const TEST262_DIR = "./test262/test262-checkout";
+const TEST262_REPO_URL = "https://github.com/tc39/test262.git";
+const TEST262_REVISION = "157b18d16b5d52501c4d75ac422d3a80bfad1c17";
+
+/**
+ * Run the test262 suite on some tests that we know are useful.
+ */
+async function main(): Promise<void> {
+  if (!(await exists(TEST262_DIR))) {
+    console.log(`Directory ${TEST262_DIR} not found, cloning a new one.`);
+    await run(`git clone ${TEST262_REPO_URL} ${TEST262_DIR}`);
+  }
+
+  // Force a specific revision so we don't get a breakage from changes to the master branch.
+  const originalCwd = process.cwd();
+  try {
+    process.chdir(TEST262_DIR);
+    await run(`git reset --hard ${TEST262_REVISION}`);
+    await run(`git clean -f`);
+  } catch (e) {
+    await run("git fetch");
+    await run(`git reset --hard ${TEST262_REVISION}`);
+    await run(`git clean -f`);
+  } finally {
+    process.chdir(originalCwd);
+  }
+
+  console.log("Running test262...");
+  const harnessStdout = (
+    await exec(`${TEST262_HARNESS} \
+    --preprocessor "./test262/test262Preprocessor.js" \
+    --reporter "json" \
+    "${TEST262_DIR}/test/language/expressions/coalesce/**/*.js" \
+    "${TEST262_DIR}/test/language/expressions/optional-chaining/**/*.js"`)
+  )[0].toString();
+
+  const harnessOutput = JSON.parse(harnessStdout);
+  let numPassed = 0;
+  let numFailed = 0;
+  for (const result of harnessOutput) {
+    if (!result.result.pass) {
+      numFailed++;
+      console.error(`${chalk.bold(chalk.bgRed("FAIL"))} ${chalk.bold(result.file)}`);
+      console.error(result.result.message);
+      console.error(`stdout:\n${result.rawResult.stdout}`);
+      console.error(`stderr:\n${result.rawResult.stderr}`);
+      console.error();
+    } else {
+      numPassed++;
+    }
+  }
+  console.log(`${numPassed} passed, ${numFailed} failed`);
+  if (numFailed > 0) {
+    process.exitCode = 1;
+  }
+}
+
+main().catch((e) => {
+  console.error("Unhandled error:");
+  console.error(e);
+  process.exitCode = 1;
+});

test262/test262Preprocessor.js

@@ -0,0 +1,34 @@
+const sucrase = require("..");
+
+/**
+ * test262-harness preprocessor documented here:
+ https://github.com/bterlson/test262-harness#preprocessor
+ */
+module.exports = function(test) {
+  // Sucrase doesn't attempt to throw SyntaxError on bad syntax, so skip those tests.
+  if (test.attrs.negative) {
+    return null;
+  }
+
+  // Sucrase assumes strict mode, so skip sloppy mode tests.
+  if (test.scenario === "default") {
+    return null;
+  }
+
+  // TCO tests seem to fail in V8 normally, so skip those.
+  if (test.attrs.features.includes("tail-call-optimization")) {
+    return null;
+  }
+
+  try {
+    test.contents = sucrase.transform(test.contents, {transforms: []}).code;
+  } catch (error) {
+    test.result = {
+      stderr: `${error.name}: ${error.message}\n`,
+      stdout: "",
+      error,
+    };
+  }
+
+  return test;
+};

tsconfig.json

@@ -30,7 +30,8 @@
     "generator",
     "script",
     "src",
-    "test"
+    "test",
+    "test262"
   ],
   "exclude": [
     "benchmark/sample",