Skip to content

Commit

Permalink
chore: use esbuild for running sources
Browse files Browse the repository at this point in the history
  • Loading branch information
merceyz committed Jan 22, 2023
1 parent 94e4069 commit 40cf4c7
Show file tree
Hide file tree
Showing 10 changed files with 256 additions and 26 deletions.
32 changes: 32 additions & 0 deletions .pnp.cjs

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Binary file not shown.
Binary file not shown.
Binary file not shown.
51 changes: 51 additions & 0 deletions bench.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
hyperfine -w 1 \
-n esbuild-cache \
--prepare "" \
"YARNPKG_TRANSPILER=esbuild node ./scripts/run-yarn.js exec yarn -v" \
-n esbuild-no-cache \
--prepare "rm -r ./node_modules/.cache/yarn" \
"YARNPKG_TRANSPILER=esbuild node ./scripts/run-yarn.js exec yarn -v" \
-n esbuild-partial-cache \
--prepare "echo '//' | tee -a packages/yarnpkg-core/sources/*.ts" \
"YARNPKG_TRANSPILER=esbuild node ./scripts/run-yarn.js exec yarn -v" \
-n babel-cache \
--prepare "" \
"YARNPKG_TRANSPILER=babel node ./scripts/run-yarn.js exec yarn -v" \
-n babel-no-cache \
--prepare "rm -r /tmp/babel" \
"YARNPKG_TRANSPILER=babel node ./scripts/run-yarn.js exec yarn -v" \
-n babel-partial-cache \
--prepare "echo '//' | tee -a packages/yarnpkg-core/sources/*.ts" \
"YARNPKG_TRANSPILER=babel node ./scripts/run-yarn.js exec yarn -v"

# Benchmark 1: esbuild-cache
# Time (mean ± σ): 4.280 s ± 0.053 s [User: 5.142 s, System: 0.432 s]
# Range (min … max): 4.245 s … 4.416 s 10 runs

# Benchmark 2: esbuild-no-cache
# Time (mean ± σ): 6.374 s ± 0.031 s [User: 5.684 s, System: 0.514 s]
# Range (min … max): 6.337 s … 6.438 s 10 runs

# Benchmark 3: esbuild-partial-cache
# Time (mean ± σ): 5.482 s ± 0.131 s [User: 5.729 s, System: 0.508 s]
# Range (min … max): 5.331 s … 5.667 s 10 runs

# Benchmark 4: babel-cache
# Time (mean ± σ): 6.626 s ± 0.145 s [User: 8.551 s, System: 0.621 s]
# Range (min … max): 6.411 s … 6.812 s 10 runs

# Benchmark 5: babel-no-cache
# Time (mean ± σ): 11.912 s ± 0.501 s [User: 17.095 s, System: 0.816 s]
# Range (min … max): 11.464 s … 12.971 s 10 runs

# Benchmark 6: babel-partial-cache
# Time (mean ± σ): 7.579 s ± 0.083 s [User: 10.345 s, System: 0.717 s]
# Range (min … max): 7.444 s … 7.710 s 10 runs

# Summary
# 'esbuild-cache' ran
# 1.28 ± 0.03 times faster than 'esbuild-partial-cache'
# 1.49 ± 0.02 times faster than 'esbuild-no-cache'
# 1.55 ± 0.04 times faster than 'babel-cache'
# 1.77 ± 0.03 times faster than 'babel-partial-cache'
# 2.78 ± 0.12 times faster than 'babel-no-cache'
3 changes: 3 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
"@babel/preset-react": "^7.18.6",
"@babel/preset-typescript": "^7.18.6",
"@babel/register": "^7.18.9",
"@cspotcode/source-map-support": "^0.8.1",
"@types/jest": "^28.1.6",
"@types/node": "^18.11.11",
"@yarnpkg/cli": "workspace:^",
Expand All @@ -22,8 +23,10 @@
"@yarnpkg/libzip": "workspace:^",
"@yarnpkg/sdks": "workspace:^",
"clipanion": "^3.2.0-rc.10",
"esbuild-wasm": "^0.16.14",
"eslint": "^8.2.0",
"jest": "^29.2.1",
"pirates": "^4.0.5",
"tslib": "^2.4.0",
"typescript": "4.9.4"
},
Expand Down
25 changes: 25 additions & 0 deletions scripts/setup-ts-execution-babel.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
const path = require(`path`);
const babel = require(`@babel/core`);
const os = require(`os`);
const root = path.dirname(__dirname);

// The cache in @babel/register never clears itself and will therefore grow
// forever causing massive slowdowns if left unchecked for a while
// this ensures a new cache key is generated every week
const weeksSinceUNIXEpoch = Math.floor(Date.now() / 604800000);

if (!process.env.BABEL_CACHE_PATH)
process.env.BABEL_CACHE_PATH = path.join(os.tmpdir(), `babel`, `.babel.${babel.version}.${babel.getEnv()}.${weeksSinceUNIXEpoch}.json`);

require(`@babel/register`)({
root,
extensions: [`.tsx`, `.ts`, `.js`],
only: [
p => {
if (p?.endsWith(`.js`))
return /packages(\\|\/)yarnpkg-pnp(\\|\/)sources(\\|\/)node/.test(p);

return true;
},
],
});
104 changes: 104 additions & 0 deletions scripts/setup-ts-execution-esbuild.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
const esbuild = require(`esbuild-wasm`);
const fs = require(`fs`);
const crypto = require(`crypto`);
const v8 = require(`v8`);
const zlib = require(`zlib`);
const path = require(`path`);
const pirates = require(`pirates`);

// Needed by the worker spawned by Esbuild
if (process.versions.pnp)
process.env.NODE_OPTIONS = `${process.env.NODE_OPTIONS || ``} -r ${JSON.stringify(require.resolve(`pnpapi`))}`;

const resolveVirtual = process.versions.pnp ? require(`pnpapi`).resolveVirtual : undefined;

const weeksSinceUNIXEpoch = Math.floor(Date.now() / 604800000);

const cache = {
version: [6, esbuild.version, weeksSinceUNIXEpoch, process.versions.node, !!process.setSourceMapsEnabled].join(`\0`),
files: new Map(),
isDirty: false,
};

const cachePath = path.join(__dirname, `../node_modules/.cache/yarn/esbuild-transpile-cache.bin`);
try {
const cacheData = v8.deserialize(zlib.gunzipSync(fs.readFileSync(cachePath)));
if (cacheData.version === cache.version) {
cache.files = cacheData.files;
}
} catch { }

function persistCache() {
if (!cache.isDirty)
return;
cache.isDirty = false;

fs.mkdirSync(path.dirname(cachePath), {recursive: true});
const tmpPath = cachePath + crypto.randomBytes(8).toString(`hex`);
fs.writeFileSync(
tmpPath,
zlib.gzipSync(
v8.serialize({
version: cache.version,
files: cache.files,
}),
{level: 1},
),
);
fs.renameSync(tmpPath, cachePath);
}

process.once(`exit`, persistCache);
process.nextTick(persistCache);

process.setSourceMapsEnabled
? process.setSourceMapsEnabled(true)
: require(`@cspotcode/source-map-support`).install({
environment: `node`,
retrieveSourceMap(filename) {
filename = resolveVirtual?.(filename) || filename;

const cacheEntry = cache.files.get(filename);
if (cacheEntry)
return {url: filename, map: cacheEntry.map};

return null;
},
});

pirates.addHook(
(sourceCode, filename) => {
filename = resolveVirtual?.(filename) || filename;

const cacheEntry = cache.files.get(filename);

if (cacheEntry?.source === sourceCode)
return cacheEntry.code;

const res = esbuild.transformSync(sourceCode, {
target: `node${process.versions.node}`,
loader: path.extname(filename).slice(1),
sourcefile: filename,
sourcemap: process.setSourceMapsEnabled ? `inline` : `both`,
platform: `node`,
format: `cjs`,
});

cache.isDirty = true;
cache.files.set(filename, {
source: sourceCode,
code: res.code,
map: res.map,
});

return res.code;
},
{
extensions: [`.tsx`, `.ts`, `.js`],
matcher(p) {
if (p?.endsWith(`.js`)) return /packages(\\|\/)yarnpkg-pnp(\\|\/)sources(\\|\/)node/.test(p);

return true;
},
},
);
34 changes: 9 additions & 25 deletions scripts/setup-ts-execution.js
Original file line number Diff line number Diff line change
@@ -1,25 +1,9 @@
const path = require(`path`);
const babel = require(`@babel/core`);
const os = require(`os`);
const root = path.dirname(__dirname);

// The cache in @babel/register never clears itself and will therefore grow
// forever causing massive slowdowns if left unchecked for a while
// this ensures a new cache key is generated every week
const weeksSinceUNIXEpoch = Math.floor(Date.now() / 604800000);

if (!process.env.BABEL_CACHE_PATH)
process.env.BABEL_CACHE_PATH = path.join(os.tmpdir(), `babel`, `.babel.${babel.version}.${babel.getEnv()}.${weeksSinceUNIXEpoch}.json`);

require(`@babel/register`)({
root,
extensions: [`.tsx`, `.ts`, `.js`],
only: [
p => {
if (p?.endsWith(`.js`))
return /packages(\\|\/)yarnpkg-pnp(\\|\/)sources(\\|\/)node/.test(p);

return true;
},
],
});
switch (process.env.YARNPKG_TRANSPILER) {
case `esbuild`:
require(`./setup-ts-execution-esbuild.js`);
break;
default:
case `babel`:
require(`./setup-ts-execution-babel.js`);
break;
}
33 changes: 32 additions & 1 deletion yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -2173,6 +2173,15 @@ __metadata:
languageName: node
linkType: hard

"@cspotcode/source-map-support@npm:^0.8.1":
version: 0.8.1
resolution: "@cspotcode/source-map-support@npm:0.8.1"
dependencies:
"@jridgewell/trace-mapping": "npm:0.3.9"
checksum: 4327d8e6e4347897f5baf265c43ff094260a3ad7b53920fa07472aa18699ba7d570e5171082e88d19e4b5cce6f35cc1666b1c8ccb8b74d67e4f482395b8c511d
languageName: node
linkType: hard

"@docsearch/css@npm:3.2.1":
version: 3.2.1
resolution: "@docsearch/css@npm:3.2.1"
Expand Down Expand Up @@ -4471,7 +4480,7 @@ __metadata:
languageName: node
linkType: hard

"@jridgewell/resolve-uri@npm:3.1.0":
"@jridgewell/resolve-uri@npm:3.1.0, @jridgewell/resolve-uri@npm:^3.0.3":
version: 3.1.0
resolution: "@jridgewell/resolve-uri@npm:3.1.0"
checksum: 6b641bb7e25bc92a9848898cc91a77a390f393f086297ec2336d911387bdd708919c418e74a22732cfc21d0e7300b94306f437d2e9de5ab58b33ebc6c39d6f9d
Expand Down Expand Up @@ -4502,6 +4511,16 @@ __metadata:
languageName: node
linkType: hard

"@jridgewell/trace-mapping@npm:0.3.9":
version: 0.3.9
resolution: "@jridgewell/trace-mapping@npm:0.3.9"
dependencies:
"@jridgewell/resolve-uri": "npm:^3.0.3"
"@jridgewell/sourcemap-codec": "npm:^1.4.10"
checksum: 542c5f0f0ae874121e9de649581f9619cc0c65e33292e1285f1233f5ff3e41e6f4f216d69a4c3f800b4d6db208ff6c710307e19e1ff170ed5304807e346e6cf9
languageName: node
linkType: hard

"@jridgewell/trace-mapping@npm:^0.3.0, @jridgewell/trace-mapping@npm:^0.3.12, @jridgewell/trace-mapping@npm:^0.3.14, @jridgewell/trace-mapping@npm:^0.3.15, @jridgewell/trace-mapping@npm:^0.3.8, @jridgewell/trace-mapping@npm:^0.3.9":
version: 0.3.17
resolution: "@jridgewell/trace-mapping@npm:0.3.17"
Expand Down Expand Up @@ -7402,6 +7421,7 @@ __metadata:
"@babel/preset-react": "npm:^7.18.6"
"@babel/preset-typescript": "npm:^7.18.6"
"@babel/register": "npm:^7.18.9"
"@cspotcode/source-map-support": "npm:^0.8.1"
"@types/jest": "npm:^28.1.6"
"@types/node": "npm:^18.11.11"
"@yarnpkg/cli": "workspace:^"
Expand All @@ -7411,8 +7431,10 @@ __metadata:
"@yarnpkg/libzip": "workspace:^"
"@yarnpkg/sdks": "workspace:^"
clipanion: "npm:^3.2.0-rc.10"
esbuild-wasm: "npm:^0.16.14"
eslint: "npm:^8.2.0"
jest: "npm:^29.2.1"
pirates: "npm:^4.0.5"
tslib: "npm:^2.4.0"
typescript: "npm:4.9.4"
dependenciesMeta:
Expand Down Expand Up @@ -13201,6 +13223,15 @@ __metadata:
languageName: node
linkType: hard

"esbuild-wasm@npm:^0.16.14":
version: 0.16.14
resolution: "esbuild-wasm@npm:0.16.14"
bin:
esbuild: bin/esbuild
checksum: f23fbc828d0088e9b9f7456cf39425c1d49476bc8f842e2cfcd520b3e7d12463c0e97dd92d2ee797318e5b50cbc4ffe7053a940667e358162e0d975e31373dad
languageName: node
linkType: hard

"esbuild-windows-32@npm:0.15.11":
version: 0.15.11
resolution: "esbuild-windows-32@npm:0.15.11"
Expand Down

0 comments on commit 40cf4c7

Please sign in to comment.