feat: inline cjs index

.changeset/rude-spiders-run.md

@@ -0,0 +1,2 @@
+---
+---

package.json

@@ -39,6 +39,7 @@
     "@typescript-eslint/parser": "4.30.0",
     "chai": "^4.2.0",
     "chai-as-promised": "^7.1.1",
+    "esbuild": "^0.19.9",
     "eslint": "7.32.0",
     "eslint-config-prettier": "8.3.0",
     "eslint-plugin-prettier": "3.4.1",

scripts/compilation/Inliner.js

@@ -0,0 +1,289 @@
+const fs = require("fs");
+const path = require("path");
+const { spawnProcess } = require("./../utils/spawn-process");
+const walk = require("./../utils/walk");
+const esbuild = require("esbuild");
+
+const root = path.join(__dirname, "..", "..");
+
+/**
+ *
+ * Inline a package as one dist file, preserves other files as re-export stubs,
+ * preserves files with react-native variants as externals.
+ *
+ */
+module.exports = class Inliner {
+  constructor(pkg) {
+    this.package = pkg;
+    this.platform = "node";
+    this.isPackage = fs.existsSync(path.join(root, "packages", pkg));
+    this.isLib = fs.existsSync(path.join(root, "lib", pkg));
+    this.isClient = !this.isPackage && !this.isLib;
+    this.subfolder = this.isPackage ? "packages" : this.isLib ? "lib" : "clients";
+
+    this.packageDirectory = path.join(root, this.subfolder, pkg);
+
+    this.outfile = path.join(root, this.subfolder, pkg, "dist-cjs", "index.js");
+  }
+
+  /**
+   * step 0: delete the dist-cjs folder.
+   */
+  async clean() {
+    await spawnProcess("yarn", ["rimraf", "./dist-cjs", "tsconfig.cjs.tsbuildinfo"], { cwd: this.packageDirectory });
+    console.log("Deleted ./dist-cjs in " + this.package);
+    return this;
+  }
+
+  /**
+   * step 1: build the default tsc dist-cjs output with dispersed files.
+   * we will need the files to be in place for stubbing.
+   */
+  async tsc() {
+    await spawnProcess("yarn", ["g:tsc", "-p", "tsconfig.cjs.json"], { cwd: this.packageDirectory });
+    console.log("Finished recompiling ./dist-cjs in " + this.package);
+    return this;
+  }
+
+  /**
+   * step 2: detect all variant files and their transitive local imports.
+   * these files will not be inlined, in order to preserve the react-native dist-cjs file replacement behavior.
+   */
+  async discoverVariants() {
+    const pkgJson = require(path.join(root, this.subfolder, this.package, "package.json"));
+    this.variantEntries = Object.entries(pkgJson["react-native"] ?? {});
+
+    for await (const file of walk(path.join(this.packageDirectory, "dist-cjs"))) {
+      if (file.endsWith(".js") && fs.existsSync(file.replace(/\.js$/, ".native.js"))) {
+        console.log("detected undeclared auto-variant", file);
+        const canonical = file.replace(/(.*?)dist-cjs\//, "./dist-cjs/").replace(/\.js$/, "");
+        const variant = canonical.replace(/(.*?)(\.js)?$/, "$1.native$2");
+
+        this.variantEntries.push([canonical, variant]);
+      }
+      if (fs.existsSync(file.replace(/\.js$/, ".browser.js"))) {
+        // not applicable to CJS?
+      }
+    }
+
+    this.transitiveVariants = [];
+
+    for (const [k, v] of this.variantEntries) {
+      for (const variantFile of [k, String(v)]) {
+        if (!variantFile.includes("dist-cjs/")) {
+          continue;
+        }
+        const keyFile = path.join(
+          this.packageDirectory,
+          "dist-cjs",
+          variantFile.replace(/(.*?)dist-cjs\//, "") + (variantFile.endsWith(".js") ? "" : ".js")
+        );
+        const keyFileContents = fs.readFileSync(keyFile, "utf-8");
+        const requireStatements = keyFileContents.matchAll(/require\("(.*?)"\)/g);
+        for (const requireStatement of requireStatements) {
+          if (requireStatement[1]?.startsWith(".")) {
+            // is relative import.
+            const key = path
+              .normalize(path.join(path.dirname(keyFile), requireStatement[1]))
+              .replace(/(.*?)dist-cjs\//, "./dist-cjs/");
+            console.log("Transitive variant file:", key);
+            this.variantEntries.push([key, key]);
+            this.transitiveVariants.push(key.replace(/(.*?)dist-cjs\//, "").replace(/(\.js)?$/, ""));
+          }
+        }
+      }
+    }
+
+    this.variantExternals = [];
+    this.variantMap = {};
+
+    for (const [k, v] of this.variantEntries) {
+      const prefix = "dist-cjs/";
+      const keyPrefixIndex = k.indexOf(prefix);
+      if (keyPrefixIndex === -1) {
+        continue;
+      }
+      const keyRelativePath = k.slice(keyPrefixIndex + prefix.length);
+      const valuePrefixIndex = String(v).indexOf(prefix);
+
+      const addJsExtension = (file) => (file.endsWith(".js") ? file : file + ".js");
+
+      if (valuePrefixIndex !== -1) {
+        const valueRelativePath = String(v).slice(valuePrefixIndex + prefix.length);
+        this.variantExternals.push(...[keyRelativePath, valueRelativePath].map(addJsExtension));
+        this.variantMap[keyRelativePath] = valueRelativePath;
+      } else {
+        this.variantExternals.push(addJsExtension(keyRelativePath));
+        this.variantMap[keyRelativePath] = v;
+      }
+    }
+
+    return this;
+  }
+
+  /**
+   * step 3: bundle the package index into dist-cjs/index.js except for node_modules
+   * and also excluding any local files that have variants for react-native.
+   */
+  async bundle() {
+    this.variantExternalsForEsBuild = this.variantExternals.map(
+      (variant) => "*/" + path.basename(variant).replace(/.js$/, "")
+    );
+
+    await esbuild.build({
+      platform: this.platform,
+      bundle: true,
+      format: "cjs",
+      mainFields: ["main"],
+      allowOverwrite: true,
+      entryPoints: [path.join(root, this.subfolder, this.package, "src", "index.ts")],
+      outfile: this.outfile,
+      external: [
+        "tslib",
+        "@aws-crypto/*",
+        "@smithy/*",
+        "@aws-sdk/*",
+        "typescript",
+        "vscode-oniguruma",
+        "pnpapi",
+        "fast-xml-parser",
+        "node_modules/*",
+        ...this.variantExternalsForEsBuild,
+      ],
+    });
+    return this;
+  }
+
+  /**
+   * step 4: rewrite all existing dist-cjs files except the index.js file.
+   * These now become re-exports of the index to preserve deep-import behavior.
+   */
+  async rewriteStubs() {
+    for await (const file of walk(path.join(this.packageDirectory, "dist-cjs"))) {
+      const relativePath = file.replace(path.join(this.packageDirectory, "dist-cjs"), "").slice(1);
+      if (relativePath === "index.js") {
+        console.log("Skipping index.js");
+        continue;
+      }
+
+      if (this.variantExternals.find((external) => relativePath.endsWith(external))) {
+        console.log("Not rewriting.", relativePath, "is variant.");
+        continue;
+      }
+
+      console.log("Rewriting", relativePath, "as index re-export stub.");
+
+      const depth = relativePath.split("/").length - 1;
+      const indexRelativePath =
+        (depth === 0
+          ? "."
+          : Array.from({ length: depth })
+              .map(() => "..")
+              .join("/")) + "/index.js";
+
+      fs.writeFileSync(file, `module.exports = require("${indexRelativePath}");`);
+    }
+
+    return this;
+  }
+
+  /**
+   * step 5: rewrite variant external imports to correct path.
+   * these externalized variants use relative imports for transitive variant files
+   * which need to be rewritten when in the index.js file.
+   */
+  async fixVariantImportPaths() {
+    this.indexContents = fs.readFileSync(this.outfile, "utf-8");
+    for (const variant of Object.keys(this.variantMap)) {
+      const basename = path.basename(variant).replace(/.js$/, "");
+      const dirname = path.dirname(variant);
+
+      const find = new RegExp(`require\\("./(.*?)/${basename}"\\)`);
+      const replace = `require("./${dirname}/${basename}")`;
+
+      this.indexContents = this.indexContents.replace(find, replace);
+
+      console.log("replacing", find, "with", replace);
+    }
+
+    fs.writeFileSync(this.outfile, this.indexContents, "utf-8");
+    return this;
+  }
+
+  /**
+   * Step 5.5, dedupe imported externals.
+   */
+  async dedupeExternals() {
+    const redundantRequireStatements = this.indexContents.matchAll(
+      /var import_([a-z_]+)(\d+) = require\("([@a-z\/-0-9]+)"\);/g
+    );
+    for (const requireStatement of redundantRequireStatements) {
+      const variableSuffix = requireStatement[1];
+      const packageName = requireStatement[3].replace("/", "\\/");
+
+      const original = this.indexContents.match(
+        new RegExp(`var import_${variableSuffix} = require\\(\"${packageName}\"\\);`)
+      );
+      if (original) {
+        let redundancyIndex = 0;
+        let misses = 0;
+
+        // perform an incremental replacement instead of a global (\d+) replacement
+        // to be safe.
+        while (true) {
+          const redundantRequire = `var import_${variableSuffix}${redundancyIndex} = require\\("${packageName}"\\);`;
+          const redundantVariable = `import_${variableSuffix}${redundancyIndex}`;
+
+          if (this.indexContents.match(new RegExp(redundantRequire))) {
+            console.log("replacing", redundantVariable);
+            this.indexContents = this.indexContents
+              .replace(new RegExp(redundantRequire, "g"), "")
+              .replace(new RegExp(redundantVariable, "g"), `import_${variableSuffix}`);
+          } else if (misses++ > 10) {
+            break;
+          }
+          redundancyIndex++;
+        }
+      }
+    }
+    fs.writeFileSync(this.outfile, this.indexContents, "utf-8");
+    return this;
+  }
+
+  /**
+   * step 6: we validate that the index.js file has a require statement
+   * for any variant files, to ensure they are not in the inlined (bundled) index.
+   */
+  async validate() {
+    this.indexContents = fs.readFileSync(this.outfile, "utf-8");
+
+    const externalsToCheck = new Set(
+      Object.keys(this.variantMap)
+        .filter((variant) => !this.transitiveVariants.includes(variant))
+        .map((variant) => path.basename(variant).replace(/.js$/, ""))
+    );
+
+    for (const line of this.indexContents.split("\n")) {
+      // we expect to see a line with require() and the variant external in it
+      if (line.includes("require(")) {
+        const checkOrder = [...externalsToCheck].sort().reverse();
+        for (const external of checkOrder) {
+          if (line.includes(external)) {
+            console.log("Inline index confirmed require() for variant external:", external);
+            externalsToCheck.delete(external);
+            continue;
+          }
+        }
+      }
+    }
+
+    if (externalsToCheck.size) {
+      throw new Error(
+        "require() statements for the following variant externals: " +
+          [...externalsToCheck].join(", ") +
+          " were not found in the index."
+      );
+    }
+    return this;
+  }
+};

scripts/inline.js

@@ -0,0 +1,40 @@
+/**
+ *
+ * Inline a package as one dist file.
+ *
+ */
+
+const fs = require("fs");
+const path = require("path");
+const Inliner = require("./compilation/Inliner");
+
+const root = path.join(__dirname, "..");
+
+const package = process.argv[2];
+
+if (!package) {
+  /**
+   * If no package is selected, this script sets all build:cjs scripts to
+   * use this inliner script instead of only tsc.
+   */
+  const packages = fs.readdirSync(path.join(root, "packages"));
+  for (const pkg of packages) {
+    const pkgJsonFilePath = path.join(root, "packages", pkg, "package.json");
+    const pkgJson = require(pkgJsonFilePath);
+
+    pkgJson.scripts["build:cjs"] = `node ../../scripts/inline ${pkg}`;
+    fs.writeFileSync(pkgJsonFilePath, JSON.stringify(pkgJson, null, 2));
+  }
+} else {
+  (async () => {
+    const inliner = new Inliner(package);
+    await inliner.clean();
+    await inliner.tsc();
+    await inliner.discoverVariants();
+    await inliner.bundle();
+    await inliner.rewriteStubs();
+    await inliner.fixVariantImportPaths();
+    await inliner.dedupeExternals();
+    await inliner.validate();
+  })();
+}

yarn.lock

@@ -1132,6 +1132,167 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@esbuild/aix-ppc64@npm:0.19.11":
+  version: 0.19.11
+  resolution: "@esbuild/aix-ppc64@npm:0.19.11"
+  conditions: os=aix & cpu=ppc64
+  languageName: node
+  linkType: hard
+
+"@esbuild/android-arm64@npm:0.19.11":
+  version: 0.19.11
+  resolution: "@esbuild/android-arm64@npm:0.19.11"
+  conditions: os=android & cpu=arm64
+  languageName: node
+  linkType: hard
+
+"@esbuild/android-arm@npm:0.19.11":
+  version: 0.19.11
+  resolution: "@esbuild/android-arm@npm:0.19.11"
+  conditions: os=android & cpu=arm
+  languageName: node
+  linkType: hard
+
+"@esbuild/android-x64@npm:0.19.11":
+  version: 0.19.11
+  resolution: "@esbuild/android-x64@npm:0.19.11"
+  conditions: os=android & cpu=x64
+  languageName: node
+  linkType: hard
+
+"@esbuild/darwin-arm64@npm:0.19.11":
+  version: 0.19.11
+  resolution: "@esbuild/darwin-arm64@npm:0.19.11"
+  conditions: os=darwin & cpu=arm64
+  languageName: node
+  linkType: hard
+
+"@esbuild/darwin-x64@npm:0.19.11":
+  version: 0.19.11
+  resolution: "@esbuild/darwin-x64@npm:0.19.11"
+  conditions: os=darwin & cpu=x64
+  languageName: node
+  linkType: hard
+
+"@esbuild/freebsd-arm64@npm:0.19.11":
+  version: 0.19.11
+  resolution: "@esbuild/freebsd-arm64@npm:0.19.11"
+  conditions: os=freebsd & cpu=arm64
+  languageName: node
+  linkType: hard
+
+"@esbuild/freebsd-x64@npm:0.19.11":
+  version: 0.19.11
+  resolution: "@esbuild/freebsd-x64@npm:0.19.11"
+  conditions: os=freebsd & cpu=x64
+  languageName: node
+  linkType: hard
+
+"@esbuild/linux-arm64@npm:0.19.11":
+  version: 0.19.11
+  resolution: "@esbuild/linux-arm64@npm:0.19.11"
+  conditions: os=linux & cpu=arm64
+  languageName: node
+  linkType: hard
+
+"@esbuild/linux-arm@npm:0.19.11":
+  version: 0.19.11
+  resolution: "@esbuild/linux-arm@npm:0.19.11"
+  conditions: os=linux & cpu=arm
+  languageName: node
+  linkType: hard
+
+"@esbuild/linux-ia32@npm:0.19.11":
+  version: 0.19.11
+  resolution: "@esbuild/linux-ia32@npm:0.19.11"
+  conditions: os=linux & cpu=ia32
+  languageName: node
+  linkType: hard
+
+"@esbuild/linux-loong64@npm:0.19.11":
+  version: 0.19.11
+  resolution: "@esbuild/linux-loong64@npm:0.19.11"
+  conditions: os=linux & cpu=loong64
+  languageName: node
+  linkType: hard
+
+"@esbuild/linux-mips64el@npm:0.19.11":
+  version: 0.19.11
+  resolution: "@esbuild/linux-mips64el@npm:0.19.11"
+  conditions: os=linux & cpu=mips64el
+  languageName: node
+  linkType: hard
+
+"@esbuild/linux-ppc64@npm:0.19.11":
+  version: 0.19.11
+  resolution: "@esbuild/linux-ppc64@npm:0.19.11"
+  conditions: os=linux & cpu=ppc64
+  languageName: node
+  linkType: hard
+
+"@esbuild/linux-riscv64@npm:0.19.11":
+  version: 0.19.11
+  resolution: "@esbuild/linux-riscv64@npm:0.19.11"
+  conditions: os=linux & cpu=riscv64
+  languageName: node
+  linkType: hard
+
+"@esbuild/linux-s390x@npm:0.19.11":
+  version: 0.19.11
+  resolution: "@esbuild/linux-s390x@npm:0.19.11"
+  conditions: os=linux & cpu=s390x
+  languageName: node
+  linkType: hard
+
+"@esbuild/linux-x64@npm:0.19.11":
+  version: 0.19.11
+  resolution: "@esbuild/linux-x64@npm:0.19.11"
+  conditions: os=linux & cpu=x64
+  languageName: node
+  linkType: hard
+
+"@esbuild/netbsd-x64@npm:0.19.11":
+  version: 0.19.11
+  resolution: "@esbuild/netbsd-x64@npm:0.19.11"
+  conditions: os=netbsd & cpu=x64
+  languageName: node
+  linkType: hard
+
+"@esbuild/openbsd-x64@npm:0.19.11":
+  version: 0.19.11
+  resolution: "@esbuild/openbsd-x64@npm:0.19.11"
+  conditions: os=openbsd & cpu=x64
+  languageName: node
+  linkType: hard
+
+"@esbuild/sunos-x64@npm:0.19.11":
+  version: 0.19.11
+  resolution: "@esbuild/sunos-x64@npm:0.19.11"
+  conditions: os=sunos & cpu=x64
+  languageName: node
+  linkType: hard
+
+"@esbuild/win32-arm64@npm:0.19.11":
+  version: 0.19.11
+  resolution: "@esbuild/win32-arm64@npm:0.19.11"
+  conditions: os=win32 & cpu=arm64
+  languageName: node
+  linkType: hard
+
+"@esbuild/win32-ia32@npm:0.19.11":
+  version: 0.19.11
+  resolution: "@esbuild/win32-ia32@npm:0.19.11"
+  conditions: os=win32 & cpu=ia32
+  languageName: node
+  linkType: hard
+
+"@esbuild/win32-x64@npm:0.19.11":
+  version: 0.19.11
+  resolution: "@esbuild/win32-x64@npm:0.19.11"
+  conditions: os=win32 & cpu=x64
+  languageName: node
+  linkType: hard
+
 "@eslint/eslintrc@npm:^0.4.3":
   version: 0.4.3
   resolution: "@eslint/eslintrc@npm:0.4.3"
@@ -5269,6 +5430,86 @@ __metadata:
   languageName: node
   linkType: hard
 
+"esbuild@npm:^0.19.9":
+  version: 0.19.11
+  resolution: "esbuild@npm:0.19.11"
+  dependencies:
+    "@esbuild/aix-ppc64": 0.19.11
+    "@esbuild/android-arm": 0.19.11
+    "@esbuild/android-arm64": 0.19.11
+    "@esbuild/android-x64": 0.19.11
+    "@esbuild/darwin-arm64": 0.19.11
+    "@esbuild/darwin-x64": 0.19.11
+    "@esbuild/freebsd-arm64": 0.19.11
+    "@esbuild/freebsd-x64": 0.19.11
+    "@esbuild/linux-arm": 0.19.11
+    "@esbuild/linux-arm64": 0.19.11
+    "@esbuild/linux-ia32": 0.19.11
+    "@esbuild/linux-loong64": 0.19.11
+    "@esbuild/linux-mips64el": 0.19.11
+    "@esbuild/linux-ppc64": 0.19.11
+    "@esbuild/linux-riscv64": 0.19.11
+    "@esbuild/linux-s390x": 0.19.11
+    "@esbuild/linux-x64": 0.19.11
+    "@esbuild/netbsd-x64": 0.19.11
+    "@esbuild/openbsd-x64": 0.19.11
+    "@esbuild/sunos-x64": 0.19.11
+    "@esbuild/win32-arm64": 0.19.11
+    "@esbuild/win32-ia32": 0.19.11
+    "@esbuild/win32-x64": 0.19.11
+  dependenciesMeta:
+    "@esbuild/aix-ppc64":
+      optional: true
+    "@esbuild/android-arm":
+      optional: true
+    "@esbuild/android-arm64":
+      optional: true
+    "@esbuild/android-x64":
+      optional: true
+    "@esbuild/darwin-arm64":
+      optional: true
+    "@esbuild/darwin-x64":
+      optional: true
+    "@esbuild/freebsd-arm64":
+      optional: true
+    "@esbuild/freebsd-x64":
+      optional: true
+    "@esbuild/linux-arm":
+      optional: true
+    "@esbuild/linux-arm64":
+      optional: true
+    "@esbuild/linux-ia32":
+      optional: true
+    "@esbuild/linux-loong64":
+      optional: true
+    "@esbuild/linux-mips64el":
+      optional: true
+    "@esbuild/linux-ppc64":
+      optional: true
+    "@esbuild/linux-riscv64":
+      optional: true
+    "@esbuild/linux-s390x":
+      optional: true
+    "@esbuild/linux-x64":
+      optional: true
+    "@esbuild/netbsd-x64":
+      optional: true
+    "@esbuild/openbsd-x64":
+      optional: true
+    "@esbuild/sunos-x64":
+      optional: true
+    "@esbuild/win32-arm64":
+      optional: true
+    "@esbuild/win32-ia32":
+      optional: true
+    "@esbuild/win32-x64":
+      optional: true
+  bin:
+    esbuild: bin/esbuild
+  checksum: ae949a796d1d06b55275ae7491ce137857468f69a93d8cc9c0943d2a701ac54e14dbb250a2ba56f2ad98283669578f1ec3bd85a4681910a5ff29a2470c3bd62c
+  languageName: node
+  linkType: hard
+
 "escalade@npm:^3.1.1":
   version: 3.1.1
   resolution: "escalade@npm:3.1.1"
@@ -9725,6 +9966,7 @@ __metadata:
     "@typescript-eslint/parser": 4.30.0
     chai: ^4.2.0
     chai-as-promised: ^7.1.1
+    esbuild: ^0.19.9
     eslint: 7.32.0
     eslint-config-prettier: 8.3.0
     eslint-plugin-prettier: 3.4.1