Skip to content

Commit fa2ce26

Browse files
kuhekuhe
committed
feat: inline cjs index
1 parent 9972041 commit fa2ce26

File tree

5 files changed

+574
-0
lines changed

5 files changed

+574
-0
lines changed

‎.changeset/rude-spiders-run.md

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
---
2+
---

‎package.json

+1
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
"@typescript-eslint/parser": "4.30.0",
4040
"chai": "^4.2.0",
4141
"chai-as-promised": "^7.1.1",
42+
"esbuild": "^0.19.9",
4243
"eslint": "7.32.0",
4344
"eslint-config-prettier": "8.3.0",
4445
"eslint-plugin-prettier": "3.4.1",

‎scripts/compilation/Inliner.js

+289
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,289 @@
1+
const fs = require("fs");
2+
const path = require("path");
3+
const { spawnProcess } = require("./../utils/spawn-process");
4+
const walk = require("./../utils/walk");
5+
const esbuild = require("esbuild");
6+
7+
const root = path.join(__dirname, "..", "..");
8+
9+
/**
10+
*
11+
* Inline a package as one dist file, preserves other files as re-export stubs,
12+
* preserves files with react-native variants as externals.
13+
*
14+
*/
15+
module.exports = class Inliner {
16+
constructor(pkg) {
17+
this.package = pkg;
18+
this.platform = "node";
19+
this.isPackage = fs.existsSync(path.join(root, "packages", pkg));
20+
this.isLib = fs.existsSync(path.join(root, "lib", pkg));
21+
this.isClient = !this.isPackage && !this.isLib;
22+
this.subfolder = this.isPackage ? "packages" : this.isLib ? "lib" : "clients";
23+
24+
this.packageDirectory = path.join(root, this.subfolder, pkg);
25+
26+
this.outfile = path.join(root, this.subfolder, pkg, "dist-cjs", "index.js");
27+
}
28+
29+
/**
30+
* step 0: delete the dist-cjs folder.
31+
*/
32+
async clean() {
33+
await spawnProcess("yarn", ["rimraf", "./dist-cjs", "tsconfig.cjs.tsbuildinfo"], { cwd: this.packageDirectory });
34+
console.log("Deleted ./dist-cjs in " + this.package);
35+
return this;
36+
}
37+
38+
/**
39+
* step 1: build the default tsc dist-cjs output with dispersed files.
40+
* we will need the files to be in place for stubbing.
41+
*/
42+
async tsc() {
43+
await spawnProcess("yarn", ["g:tsc", "-p", "tsconfig.cjs.json"], { cwd: this.packageDirectory });
44+
console.log("Finished recompiling ./dist-cjs in " + this.package);
45+
return this;
46+
}
47+
48+
/**
49+
* step 2: detect all variant files and their transitive local imports.
50+
* these files will not be inlined, in order to preserve the react-native dist-cjs file replacement behavior.
51+
*/
52+
async discoverVariants() {
53+
const pkgJson = require(path.join(root, this.subfolder, this.package, "package.json"));
54+
this.variantEntries = Object.entries(pkgJson["react-native"] ?? {});
55+
56+
for await (const file of walk(path.join(this.packageDirectory, "dist-cjs"))) {
57+
if (file.endsWith(".js") && fs.existsSync(file.replace(/\.js$/, ".native.js"))) {
58+
console.log("detected undeclared auto-variant", file);
59+
const canonical = file.replace(/(.*?)dist-cjs\//, "./dist-cjs/").replace(/\.js$/, "");
60+
const variant = canonical.replace(/(.*?)(\.js)?$/, "$1.native$2");
61+
62+
this.variantEntries.push([canonical, variant]);
63+
}
64+
if (fs.existsSync(file.replace(/\.js$/, ".browser.js"))) {
65+
// not applicable to CJS?
66+
}
67+
}
68+
69+
this.transitiveVariants = [];
70+
71+
for (const [k, v] of this.variantEntries) {
72+
for (const variantFile of [k, String(v)]) {
73+
if (!variantFile.includes("dist-cjs/")) {
74+
continue;
75+
}
76+
const keyFile = path.join(
77+
this.packageDirectory,
78+
"dist-cjs",
79+
variantFile.replace(/(.*?)dist-cjs\//, "") + (variantFile.endsWith(".js") ? "" : ".js")
80+
);
81+
const keyFileContents = fs.readFileSync(keyFile, "utf-8");
82+
const requireStatements = keyFileContents.matchAll(/require\("(.*?)"\)/g);
83+
for (const requireStatement of requireStatements) {
84+
if (requireStatement[1]?.startsWith(".")) {
85+
// is relative import.
86+
const key = path
87+
.normalize(path.join(path.dirname(keyFile), requireStatement[1]))
88+
.replace(/(.*?)dist-cjs\//, "./dist-cjs/");
89+
console.log("Transitive variant file:", key);
90+
this.variantEntries.push([key, key]);
91+
this.transitiveVariants.push(key.replace(/(.*?)dist-cjs\//, "").replace(/(\.js)?$/, ""));
92+
}
93+
}
94+
}
95+
}
96+
97+
this.variantExternals = [];
98+
this.variantMap = {};
99+
100+
for (const [k, v] of this.variantEntries) {
101+
const prefix = "dist-cjs/";
102+
const keyPrefixIndex = k.indexOf(prefix);
103+
if (keyPrefixIndex === -1) {
104+
continue;
105+
}
106+
const keyRelativePath = k.slice(keyPrefixIndex + prefix.length);
107+
const valuePrefixIndex = String(v).indexOf(prefix);
108+
109+
const addJsExtension = (file) => (file.endsWith(".js") ? file : file + ".js");
110+
111+
if (valuePrefixIndex !== -1) {
112+
const valueRelativePath = String(v).slice(valuePrefixIndex + prefix.length);
113+
this.variantExternals.push(...[keyRelativePath, valueRelativePath].map(addJsExtension));
114+
this.variantMap[keyRelativePath] = valueRelativePath;
115+
} else {
116+
this.variantExternals.push(addJsExtension(keyRelativePath));
117+
this.variantMap[keyRelativePath] = v;
118+
}
119+
}
120+
121+
return this;
122+
}
123+
124+
/**
125+
* step 3: bundle the package index into dist-cjs/index.js except for node_modules
126+
* and also excluding any local files that have variants for react-native.
127+
*/
128+
async bundle() {
129+
this.variantExternalsForEsBuild = this.variantExternals.map(
130+
(variant) => "*/" + path.basename(variant).replace(/.js$/, "")
131+
);
132+
133+
await esbuild.build({
134+
platform: this.platform,
135+
bundle: true,
136+
format: "cjs",
137+
mainFields: ["main"],
138+
allowOverwrite: true,
139+
entryPoints: [path.join(root, this.subfolder, this.package, "src", "index.ts")],
140+
outfile: this.outfile,
141+
external: [
142+
"tslib",
143+
"@aws-crypto/*",
144+
"@smithy/*",
145+
"@aws-sdk/*",
146+
"typescript",
147+
"vscode-oniguruma",
148+
"pnpapi",
149+
"fast-xml-parser",
150+
"node_modules/*",
151+
...this.variantExternalsForEsBuild,
152+
],
153+
});
154+
return this;
155+
}
156+
157+
/**
158+
* step 4: rewrite all existing dist-cjs files except the index.js file.
159+
* These now become re-exports of the index to preserve deep-import behavior.
160+
*/
161+
async rewriteStubs() {
162+
for await (const file of walk(path.join(this.packageDirectory, "dist-cjs"))) {
163+
const relativePath = file.replace(path.join(this.packageDirectory, "dist-cjs"), "").slice(1);
164+
if (relativePath === "index.js") {
165+
console.log("Skipping index.js");
166+
continue;
167+
}
168+
169+
if (this.variantExternals.find((external) => relativePath.endsWith(external))) {
170+
console.log("Not rewriting.", relativePath, "is variant.");
171+
continue;
172+
}
173+
174+
console.log("Rewriting", relativePath, "as index re-export stub.");
175+
176+
const depth = relativePath.split("/").length - 1;
177+
const indexRelativePath =
178+
(depth === 0
179+
? "."
180+
: Array.from({ length: depth })
181+
.map(() => "..")
182+
.join("/")) + "/index.js";
183+
184+
fs.writeFileSync(file, `module.exports = require("${indexRelativePath}");`);
185+
}
186+
187+
return this;
188+
}
189+
190+
/**
191+
* step 5: rewrite variant external imports to correct path.
192+
* these externalized variants use relative imports for transitive variant files
193+
* which need to be rewritten when in the index.js file.
194+
*/
195+
async fixVariantImportPaths() {
196+
this.indexContents = fs.readFileSync(this.outfile, "utf-8");
197+
for (const variant of Object.keys(this.variantMap)) {
198+
const basename = path.basename(variant).replace(/.js$/, "");
199+
const dirname = path.dirname(variant);
200+
201+
const find = new RegExp(`require\\("./(.*?)/${basename}"\\)`);
202+
const replace = `require("./${dirname}/${basename}")`;
203+
204+
this.indexContents = this.indexContents.replace(find, replace);
205+
206+
console.log("replacing", find, "with", replace);
207+
}
208+
209+
fs.writeFileSync(this.outfile, this.indexContents, "utf-8");
210+
return this;
211+
}
212+
213+
/**
214+
* Step 5.5, dedupe imported externals.
215+
*/
216+
async dedupeExternals() {
217+
const redundantRequireStatements = this.indexContents.matchAll(
218+
/var import_([a-z_]+)(\d+) = require\("([@a-z\/-0-9]+)"\);/g
219+
);
220+
for (const requireStatement of redundantRequireStatements) {
221+
const variableSuffix = requireStatement[1];
222+
const packageName = requireStatement[3].replace("/", "\\/");
223+
224+
const original = this.indexContents.match(
225+
new RegExp(`var import_${variableSuffix} = require\\(\"${packageName}\"\\);`)
226+
);
227+
if (original) {
228+
let redundancyIndex = 0;
229+
let misses = 0;
230+
231+
// perform an incremental replacement instead of a global (\d+) replacement
232+
// to be safe.
233+
while (true) {
234+
const redundantRequire = `var import_${variableSuffix}${redundancyIndex} = require\\("${packageName}"\\);`;
235+
const redundantVariable = `import_${variableSuffix}${redundancyIndex}`;
236+
237+
if (this.indexContents.match(new RegExp(redundantRequire))) {
238+
console.log("replacing", redundantVariable);
239+
this.indexContents = this.indexContents
240+
.replace(new RegExp(redundantRequire, "g"), "")
241+
.replace(new RegExp(redundantVariable, "g"), `import_${variableSuffix}`);
242+
} else if (misses++ > 10) {
243+
break;
244+
}
245+
redundancyIndex++;
246+
}
247+
}
248+
}
249+
fs.writeFileSync(this.outfile, this.indexContents, "utf-8");
250+
return this;
251+
}
252+
253+
/**
254+
* step 6: we validate that the index.js file has a require statement
255+
* for any variant files, to ensure they are not in the inlined (bundled) index.
256+
*/
257+
async validate() {
258+
this.indexContents = fs.readFileSync(this.outfile, "utf-8");
259+
260+
const externalsToCheck = new Set(
261+
Object.keys(this.variantMap)
262+
.filter((variant) => !this.transitiveVariants.includes(variant))
263+
.map((variant) => path.basename(variant).replace(/.js$/, ""))
264+
);
265+
266+
for (const line of this.indexContents.split("\n")) {
267+
// we expect to see a line with require() and the variant external in it
268+
if (line.includes("require(")) {
269+
const checkOrder = [...externalsToCheck].sort().reverse();
270+
for (const external of checkOrder) {
271+
if (line.includes(external)) {
272+
console.log("Inline index confirmed require() for variant external:", external);
273+
externalsToCheck.delete(external);
274+
continue;
275+
}
276+
}
277+
}
278+
}
279+
280+
if (externalsToCheck.size) {
281+
throw new Error(
282+
"require() statements for the following variant externals: " +
283+
[...externalsToCheck].join(", ") +
284+
" were not found in the index."
285+
);
286+
}
287+
return this;
288+
}
289+
};

‎scripts/inline.js

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/**
2+
*
3+
* Inline a package as one dist file.
4+
*
5+
*/
6+
7+
const fs = require("fs");
8+
const path = require("path");
9+
const Inliner = require("./compilation/Inliner");
10+
11+
const root = path.join(__dirname, "..");
12+
13+
const package = process.argv[2];
14+
15+
if (!package) {
16+
/**
17+
* If no package is selected, this script sets all build:cjs scripts to
18+
* use this inliner script instead of only tsc.
19+
*/
20+
const packages = fs.readdirSync(path.join(root, "packages"));
21+
for (const pkg of packages) {
22+
const pkgJsonFilePath = path.join(root, "packages", pkg, "package.json");
23+
const pkgJson = require(pkgJsonFilePath);
24+
25+
pkgJson.scripts["build:cjs"] = `node ../../scripts/inline ${pkg}`;
26+
fs.writeFileSync(pkgJsonFilePath, JSON.stringify(pkgJson, null, 2));
27+
}
28+
} else {
29+
(async () => {
30+
const inliner = new Inliner(package);
31+
await inliner.clean();
32+
await inliner.tsc();
33+
await inliner.discoverVariants();
34+
await inliner.bundle();
35+
await inliner.rewriteStubs();
36+
await inliner.fixVariantImportPaths();
37+
await inliner.dedupeExternals();
38+
await inliner.validate();
39+
})();
40+
}

0 commit comments

Comments
 (0)