From 5dc816ef5c622c19bf683b4f8ea6bbf77d51934c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=A7=91=F0=9F=8F=BB=E2=80=8D=F0=9F=92=BB=20Romain=20M?= =?UTF-8?q?arcadier?= Date: Fri, 26 Aug 2022 17:48:30 +0200 Subject: [PATCH 1/6] feat(kernel): experimental runtime package cache Adds an experimental (hence opt-in) feature that caches the contents of loaded libraries in a directory that persists between executions, in order to spare the time it takes to extract the tarballs. When this feature is enabled, packages present in the cache will be used as-is (i.e: they are not checked for tampering) instead of being extracted from the tarball. The cache is keyed on: - The hash of the tarball - The name of the library - The version of the library Objects in the cache will expire if they are not used for 30 days, and are subsequently removed from disk (this avoids a cache growing extremely large over time). In order to enable the feature, the following environment variables are used: - `JSII_RUNTIME_PACKAGE_CACHE` must be set to `enabled` in order for the package cache to be active at all; - `JSII_RUNTIME_PACKAGE_CACHE_ROOT` can be used to change which directory is used as a cache root. It defaults to: * On MacOS: `$HOME/Library/Caches/com.amazonaws.jsii` * On Linux: `$HOME/.cache/aws/jsii/package-cache` * On Windows: `%LOCALAPPDATA%\AWS\jsii\package-cache` * On other platforms: `$TMP/aws-jsii-package-cache` - `JSII_RUNTIME_PACKAGE_CACHE_TTL` can be used to change the default time entries will remain in cache before expiring if they are not used. This defaults to 30 days, and the value is expressed in days. Set to `0` to immediately expire all the cache's content. When troubleshooting load performance, it is possible to obtain timing data for some critical parts of the library load process within the jsii kernel by setting `JSII_DEBUG_TIMING` environment variable. Related to #3389 --- packages/@jsii/kernel/package.json | 2 + packages/@jsii/kernel/src/kernel.test.ts | 11 + packages/@jsii/kernel/src/kernel.ts | 90 ++++++-- packages/@jsii/kernel/src/link.ts | 26 +++ packages/@jsii/kernel/src/objects.ts | 26 ++- .../src/tar-cache/default-cache-root.ts | 27 +++ .../@jsii/kernel/src/tar-cache/digest-file.ts | 27 +++ .../@jsii/kernel/src/tar-cache/disk-cache.ts | 200 ++++++++++++++++++ packages/@jsii/kernel/src/tar-cache/index.ts | 119 +++++++++++ packages/@jsii/runtime/lib/host.ts | 9 +- packages/@jsii/runtime/lib/program.ts | 3 +- yarn.lock | 12 ++ 12 files changed, 528 insertions(+), 24 deletions(-) create mode 100644 packages/@jsii/kernel/src/link.ts create mode 100644 packages/@jsii/kernel/src/tar-cache/default-cache-root.ts create mode 100644 packages/@jsii/kernel/src/tar-cache/digest-file.ts create mode 100644 packages/@jsii/kernel/src/tar-cache/disk-cache.ts create mode 100644 packages/@jsii/kernel/src/tar-cache/index.ts diff --git a/packages/@jsii/kernel/package.json b/packages/@jsii/kernel/package.json index 3c3474a21e..76e82ed9a8 100644 --- a/packages/@jsii/kernel/package.json +++ b/packages/@jsii/kernel/package.json @@ -33,12 +33,14 @@ "dependencies": { "@jsii/spec": "^0.0.0", "fs-extra": "^10.1.0", + "lockfile": "^1.0.4", "tar": "^6.1.11" }, "devDependencies": { "@scope/jsii-calc-base": "^0.0.0", "@scope/jsii-calc-lib": "^0.0.0", "@types/fs-extra": "^9.0.13", + "@types/lockfile": "^1.0.2", "@types/tar": "^6.1.2", "jest-expect-message": "^1.0.2", "jsii-build-tools": "^0.0.0", diff --git a/packages/@jsii/kernel/src/kernel.test.ts b/packages/@jsii/kernel/src/kernel.test.ts index 113a2d1692..9365846744 100644 --- a/packages/@jsii/kernel/src/kernel.test.ts +++ b/packages/@jsii/kernel/src/kernel.test.ts @@ -16,6 +16,9 @@ import { } from './api'; import { Kernel } from './kernel'; import { closeRecording, recordInteraction } from './recording'; +import * as tar from './tar-cache'; +import { defaultCacheRoot } from './tar-cache/default-cache-root'; +import { DiskCache } from './tar-cache/disk-cache'; /* eslint-disable require-atomic-updates */ @@ -49,6 +52,11 @@ if (recordingOutput) { console.error(`JSII_RECORD=${recordingOutput}`); } +afterAll(() => { + // Jest prevents execution of "beforeExit" events. + DiskCache.inDirectory(defaultCacheRoot()).pruneExpiredEntries(); +}); + function defineTest( name: string, method: (sandbox: Kernel) => Promise | any, @@ -2147,6 +2155,9 @@ defineTest('invokeBinScript() return output', (sandbox) => { const testNames: { [name: string]: boolean } = {}; async function createCalculatorSandbox(name: string) { + // Run half the tests with cache, half without cache... so we test both. + tar.setPackageCacheEnabled(!tar.getPackageCacheEnabled()); + if (name in testNames) { throw new Error(`Duplicate test name: ${name}`); } diff --git a/packages/@jsii/kernel/src/kernel.ts b/packages/@jsii/kernel/src/kernel.ts index a24790a128..7426dab9da 100644 --- a/packages/@jsii/kernel/src/kernel.ts +++ b/packages/@jsii/kernel/src/kernel.ts @@ -1,23 +1,29 @@ import * as spec from '@jsii/spec'; import { loadAssemblyFromPath } from '@jsii/spec'; import * as cp from 'child_process'; +import { renameSync } from 'fs'; import * as fs from 'fs-extra'; import { createRequire } from 'module'; import * as os from 'os'; import * as path from 'path'; -import * as tar from 'tar'; import * as api from './api'; import { TOKEN_REF } from './api'; +import { link } from './link'; import { jsiiTypeFqn, ObjectTable, tagJsiiConstructor } from './objects'; import * as onExit from './on-exit'; import * as wire from './serialization'; +import * as tar from './tar-cache'; export class Kernel { /** * Set to true for verbose debugging. */ public traceEnabled = false; + /** + * Set to true for timing data to be emitted. + */ + public debugTimingEnabled = false; private readonly assemblies = new Map(); private readonly objects = new ObjectTable(this._typeInfoForFqn.bind(this)); @@ -40,6 +46,13 @@ export class Kernel { public constructor(public callbackHandler: (callback: api.Callback) => any) {} public load(req: api.LoadRequest): api.LoadResponse { + return this._debugTime( + () => this._load(req), + `load(${JSON.stringify(req, null, 2)})`, + ); + } + + private _load(req: api.LoadRequest): api.LoadResponse { this._debug('load', req); if ('assembly' in req) { @@ -74,21 +87,41 @@ export class Kernel { }; } - // Create the install directory (there may be several path components for @scoped/packages) - fs.mkdirpSync(packageDir); - // Force umask to have npm-install-like permissions const originalUmask = process.umask(0o022); try { // untar the archive to its final location - tar.extract({ - cwd: packageDir, - file: req.tarball, - strict: true, - strip: 1, // Removes the 'package/' path element from entries - sync: true, - unlink: true, - }); + const { path: extractedTo, cache } = this._debugTime( + () => + tar.extract( + req.tarball, + { + strict: true, + strip: 1, // Removes the 'package/' path element from entries + unlink: true, + }, + req.name, + req.version, + ), + `tar.extract(${req.tarball}) => ${packageDir}`, + ); + + // Create the install directory (there may be several path components for @scoped/packages) + fs.mkdirSync(path.dirname(packageDir), { recursive: true }); + if (cache != null) { + this._debug( + `Package cache enabled, extraction resulted in a cache ${cache}`, + ); + + // Link the package into place. + this._debugTime( + () => link(extractedTo, packageDir), + `link(${extractedTo}, ${packageDir})`, + ); + } else { + // This is not from cache, so we move it around instead of copying. + renameSync(extractedTo, packageDir); + } } finally { // Reset umask to the initial value process.umask(originalUmask); @@ -97,15 +130,26 @@ export class Kernel { // read .jsii metadata from the root of the package let assmSpec; try { - assmSpec = loadAssemblyFromPath(packageDir); + assmSpec = this._debugTime( + () => loadAssemblyFromPath(packageDir), + `loadAssemblyFromPath(${packageDir})`, + ); } catch (e: any) { throw new Error(`Error for package tarball ${req.tarball}: ${e.message}`); } // load the module and capture its closure - const closure = this.require!(packageDir); + const closure = this._debugTime( + () => this.require!(packageDir), + `require(${packageDir})`, + ); const assm = new Assembly(assmSpec, closure); - this._addAssembly(assm); + this._debugTime( + () => this._addAssembly(assm), + `registerAssembly({ name: ${assm.metadata.name}, types: ${ + Object.keys(assm.metadata.types ?? {}).length + } })`, + ); return { assembly: assmSpec.name, @@ -511,7 +555,7 @@ export class Kernel { case spec.TypeKind.Class: case spec.TypeKind.Enum: const constructor = this._findSymbol(fqn); - tagJsiiConstructor(constructor, fqn); + tagJsiiConstructor(constructor, fqn, assm.metadata.version); } } } @@ -1212,6 +1256,20 @@ export class Kernel { } } + private _debugTime(cb: () => T, label: string): T { + const fullLabel = `[@jsii/kernel:timing] ${label}`; + if (this.debugTimingEnabled) { + console.time(fullLabel); + } + try { + return cb(); + } finally { + if (this.debugTimingEnabled) { + console.timeEnd(fullLabel); + } + } + } + /** * Ensures that `fn` is called and defends against beginning to invoke * async methods until fn finishes (successfully or not). diff --git a/packages/@jsii/kernel/src/link.ts b/packages/@jsii/kernel/src/link.ts new file mode 100644 index 0000000000..4052b5895a --- /dev/null +++ b/packages/@jsii/kernel/src/link.ts @@ -0,0 +1,26 @@ +import { copyFileSync, linkSync, mkdirSync, readdirSync, statSync } from 'fs'; +import { join } from 'path'; + +/** + * Creates directories containing hard links if possible, and falls back on + * copy otherwise. + * + * @param existing is the original file or directory to link. + * @param destination is the nbew file or directory to create. + */ +export function link(existing: string, destination: string): void { + const stat = statSync(existing); + if (!stat.isDirectory()) { + try { + linkSync(existing, destination); + } catch { + copyFileSync(existing, destination); + } + return; + } + + mkdirSync(destination, { recursive: true }); + for (const file of readdirSync(existing)) { + link(join(existing, file), join(destination, file)); + } +} diff --git a/packages/@jsii/kernel/src/objects.ts b/packages/@jsii/kernel/src/objects.ts index c191c222bb..293732ea48 100644 --- a/packages/@jsii/kernel/src/objects.ts +++ b/packages/@jsii/kernel/src/objects.ts @@ -16,7 +16,16 @@ const IFACES_SYMBOL = Symbol.for('$__jsii__interfaces__$'); /** * Symbol we use to tag the constructor of a JSII class */ -const JSII_SYMBOL = Symbol.for('__jsii__'); +const JSII_RTTI_SYMBOL = Symbol.for('jsii.rtti'); + +interface ManagedConstructor { + readonly [JSII_RTTI_SYMBOL]: { + readonly fqn: string; + readonly version: string; + }; +} + +type MaybeManagedConstructor = Partial; /** * Get the JSII fqn for an object (if available) @@ -26,7 +35,7 @@ const JSII_SYMBOL = Symbol.for('__jsii__'); * information. */ export function jsiiTypeFqn(obj: any): string | undefined { - return obj.constructor[JSII_SYMBOL]?.fqn; + return (obj.constructor as MaybeManagedConstructor)[JSII_RTTI_SYMBOL]?.fqn; } /** @@ -86,12 +95,19 @@ function tagObject(obj: unknown, objid: string, interfaces?: string[]) { /** * Set the JSII FQN for classes produced by a given constructor */ -export function tagJsiiConstructor(constructor: any, fqn: string) { - Object.defineProperty(constructor, JSII_SYMBOL, { +export function tagJsiiConstructor( + constructor: any, + fqn: string, + version: string, +) { + if (Object.prototype.hasOwnProperty.call(constructor, JSII_RTTI_SYMBOL)) { + return; + } + Object.defineProperty(constructor, JSII_RTTI_SYMBOL, { configurable: false, enumerable: false, writable: false, - value: { fqn }, + value: { fqn, version }, }); } diff --git a/packages/@jsii/kernel/src/tar-cache/default-cache-root.ts b/packages/@jsii/kernel/src/tar-cache/default-cache-root.ts new file mode 100644 index 0000000000..d61f57a404 --- /dev/null +++ b/packages/@jsii/kernel/src/tar-cache/default-cache-root.ts @@ -0,0 +1,27 @@ +import { tmpdir } from 'os'; +import { join } from 'path'; + +export function defaultCacheRoot(): string { + switch (process.platform) { + case 'darwin': + if (process.env.HOME) + return join( + process.env.HOME, + 'Library', + 'Caches', + 'com.amazonaws.jsii', + ); + break; + case 'linux': + if (process.env.HOME) + return join(process.env.HOME, '.cache', 'aws', 'jsii', 'package-cache'); + break; + case 'win32': + if (process.env.LOCALAPPDATA) + return join(process.env.LOCALAPPDATA, 'AWS', 'jsii', 'package-cache'); + break; + default: + // Fall back on putting in tmpdir() + } + return join(tmpdir(), 'aws-jsii-package-cache'); +} diff --git a/packages/@jsii/kernel/src/tar-cache/digest-file.ts b/packages/@jsii/kernel/src/tar-cache/digest-file.ts new file mode 100644 index 0000000000..f208a5072e --- /dev/null +++ b/packages/@jsii/kernel/src/tar-cache/digest-file.ts @@ -0,0 +1,27 @@ +import { createHash } from 'crypto'; +import { openSync, readSync, closeSync } from 'fs'; + +const ALGORITHM = 'sha256'; + +export function digestFile( + path: string, + ...comments: readonly string[] +): Buffer { + const hash = createHash(ALGORITHM); + + const buffer = Buffer.alloc(16_384); + const fd = openSync(path, 'r'); + try { + let bytesRead = 0; + while ((bytesRead = readSync(fd, buffer)) > 0) { + hash.update(buffer.slice(0, bytesRead)); + } + for (const comment of comments) { + hash.update('\0'); + hash.update(comment); + } + return hash.digest(); + } finally { + closeSync(fd); + } +} diff --git a/packages/@jsii/kernel/src/tar-cache/disk-cache.ts b/packages/@jsii/kernel/src/tar-cache/disk-cache.ts new file mode 100644 index 0000000000..04cfc4fc56 --- /dev/null +++ b/packages/@jsii/kernel/src/tar-cache/disk-cache.ts @@ -0,0 +1,200 @@ +import { + existsSync, + mkdirSync, + readdirSync, + realpathSync, + rmdirSync, + rmSync, + statSync, + utimesSync, + writeFileSync, +} from 'fs'; +import { lockSync, unlockSync } from 'lockfile'; +import { dirname, join } from 'path'; + +import { digestFile } from './digest-file'; + +const MARKER_FILE_NAME = '.jsii-runtime-package-cache'; + +const ONE_DAY_IN_MS = 86_400_000; +const PRUNE_AFTER_MILLISECONDS = process.env.JSII_RUNTIME_PACKAGE_CACHE_TTL + ? parseInt(process.env.JSII_RUNTIME_PACKAGE_CACHE_TTL, 10) * ONE_DAY_IN_MS + : 30 * ONE_DAY_IN_MS; + +export class DiskCache { + private static readonly CACHE = new Map(); + + public static inDirectory(path: string): DiskCache { + const didCreate = mkdirSync(path, { recursive: true }) != null; + if (didCreate && process.platform === 'darwin') { + // Mark the directories for no iCloud sync, no Spotlight indexing, no TimeMachine backup + // @see https://michaelbach.de/2019/03/19/MacOS-nosync-noindex-nobackup.html + writeFileSync(join(path, '.nobackup'), ''); + writeFileSync(join(path, '.noindex'), ''); + writeFileSync(join(path, '.nosync'), ''); + } + + path = realpathSync(path); + if (!this.CACHE.has(path)) { + this.CACHE.set(path, new DiskCache(path)); + } + return this.CACHE.get(path)!; + } + + readonly #root: string; + + private constructor(root: string) { + this.#root = root; + process.once('beforeExit', () => this.pruneExpiredEntries()); + } + + public entryFor(path: string, ...comments: readonly string[]) { + const rawDigest = digestFile(path, ...comments); + return new Entry( + join( + this.#root, + ...comments.flatMap((s) => + s.replace(/[^@a-z0-9_.\\/-]+/g, '_').split(/[\\/]+/), + ), + rawDigest.toString('hex'), + ), + ); + } + + public pruneExpiredEntries() { + const cutOff = new Date(Date.now() - PRUNE_AFTER_MILLISECONDS); + for (const entry of this.entries()) { + if (entry.atime < cutOff) { + entry.lock((lockedEntry) => { + // Check again in case it's been accessed which we waited for the lock... + if (entry.atime > cutOff) { + return; + } + lockedEntry.delete(); + }); + } + } + + for (const dir of directoriesUnder(this.#root, true)) { + if (process.platform === 'darwin') { + try { + rmSync(join(dir, '.DS_Store'), { force: true }); + } catch { + // Ignore errors... + } + } + if (readdirSync(dir).length === 0) { + try { + rmdirSync(dir); + } catch { + // Ignore errors, directory may no longer be empty... + } + } + } + } + + private *entries(): Generator { + yield* inDirectory(this.#root); + + function* inDirectory(dir: string): Generator { + if (existsSync(join(dir, MARKER_FILE_NAME))) { + return yield new Entry(dir); + } + for (const file of directoriesUnder(dir)) { + yield* inDirectory(file); + } + } + } +} + +export class Entry { + public constructor(public readonly path: string) {} + + public get atime(): Date { + try { + const stat = statSync(this.markerFile); + return stat.atime; + } catch (err: any) { + if (err.code !== 'ENOENT') { + throw err; + } + return new Date(0); + } + } + + public get pathExists() { + return existsSync(this.path); + } + + private get lockFile(): string { + return `${this.path}.lock`; + } + + private get markerFile(): string { + return join(this.path, MARKER_FILE_NAME); + } + + public lock(cb: (entry: LockedEntry) => T): T { + mkdirSync(dirname(this.path), { recursive: true }); + lockSync(this.lockFile, { retries: 12, stale: 5_000 }); + let disposed = false; + try { + return cb({ + delete: () => { + if (disposed) { + throw new Error( + `Cannot delete ${this.path} once the lock block as returned!`, + ); + } + rmSync(this.path, { force: true, recursive: true }); + }, + touch: () => { + if (disposed) { + throw new Error( + `Cannot touch ${this.path} once the lock block as returned!`, + ); + } + if (this.pathExists) { + if (existsSync(this.markerFile)) { + const now = new Date(); + utimesSync(this.markerFile, now, now); + } else { + writeFileSync(this.markerFile, ''); + } + } + }, + }); + } finally { + disposed = true; + unlockSync(this.lockFile); + } + } +} + +export interface LockedEntry { + delete(): void; + touch(): void; +} + +function* directoriesUnder( + root: string, + recursive = false, + ignoreErrors = true, +): Generator { + for (const file of readdirSync(root)) { + const path = join(root, file); + try { + const stat = statSync(path); + if (stat.isDirectory()) { + if (recursive) { + yield* directoriesUnder(path, recursive, ignoreErrors); + } + yield path; + } + } catch (error) { + if (!ignoreErrors) { + throw error; + } + } + } +} diff --git a/packages/@jsii/kernel/src/tar-cache/index.ts b/packages/@jsii/kernel/src/tar-cache/index.ts new file mode 100644 index 0000000000..4f68d82ad8 --- /dev/null +++ b/packages/@jsii/kernel/src/tar-cache/index.ts @@ -0,0 +1,119 @@ +import { mkdirSync, mkdtempSync, renameSync, rmSync } from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import * as tar from 'tar'; + +import { defaultCacheRoot } from './default-cache-root'; +import { DiskCache } from './disk-cache'; + +export type ExtractOptions = Omit< + tar.ExtractOptions & tar.FileOptions, + 'file' | 'cwd' +>; + +export interface ExtractResult { + /** + * The path in which the extracted files are located + */ + readonly path: string; + + /** + * When `'hit'`, the data was already present in cache and was returned from + * cache. + * + * When `'miss'`, the data was extracted into the caache and returned from + * cache. + * + * When `undefined`, the cache is not enabled. + */ + readonly cache?: 'hit' | 'miss'; +} + +let packageCacheEnabled = + process.env.JSII_RUNTIME_PACKAGE_CACHE?.toLocaleUpperCase() === 'enabled'; + +/** + * Extracts the content of a tarball, possibly caching it on disk. + * + * @param file is the path to the tarball to be extracted. + * @param options are options to pass to `tar.extract` + * @param comments are included in the cache key, when caching is enabled. + * + * @returns the result of the extraction. + */ +export function extract( + file: string, + options: ExtractOptions, + ...comments: readonly string[] +): ExtractResult { + return (packageCacheEnabled ? extractToCache : extractToTemporary)( + file, + options, + ...comments, + ); +} + +function extractToCache( + file: string, + options: ExtractOptions = {}, + ...comments: readonly string[] +): { path: string; cache: 'hit' | 'miss' } { + const cacheRoot = + process.env.JSII_RUNTIME_PACKAGE_CACHE_ROOT ?? defaultCacheRoot(); + const cache = DiskCache.inDirectory(cacheRoot); + + const entry = cache.entryFor(file, ...comments); + return entry.lock((lock) => { + let cache: 'hit' | 'miss' = 'hit'; + if (!entry.pathExists) { + const tmpPath = `${entry.path}.tmp`; + mkdirSync(tmpPath, { recursive: true }); + try { + untarInto({ + ...options, + cwd: tmpPath, + file, + }); + renameSync(tmpPath, entry.path); + } catch (error) { + rmSync(entry.path, { force: true, recursive: true }); + throw error; + } + cache = 'miss'; + } + lock.touch(); + return { path: entry.path, cache }; + }); +} + +function extractToTemporary( + file: string, + options: ExtractOptions = {}, +): { path: string } { + const path = mkdtempSync(join(tmpdir(), 'jsii-runtime-untar-')); + + untarInto({ ...options, cwd: path, file }); + + return { path }; +} + +function untarInto( + options: tar.ExtractOptions & tar.FileOptions & { cwd: string }, +) { + try { + tar.extract({ ...options, sync: true }); + } catch (error) { + rmSync(options.cwd, { force: true, recursive: true }); + throw error; + } +} + +/** @internal */ +export function getPackageCacheEnabled(): boolean { + return packageCacheEnabled; +} + +/** @internal */ +export function setPackageCacheEnabled(value: boolean) { + packageCacheEnabled = value; +} diff --git a/packages/@jsii/runtime/lib/host.ts b/packages/@jsii/runtime/lib/host.ts index 9522d22e01..44a38dec7b 100644 --- a/packages/@jsii/runtime/lib/host.ts +++ b/packages/@jsii/runtime/lib/host.ts @@ -9,9 +9,14 @@ export class KernelHost { public constructor( private readonly inout: IInputOutput, - private readonly opts: { debug?: boolean; noStack?: boolean } = {}, + private readonly opts: { + debug?: boolean; + debugTiming?: boolean; + noStack?: boolean; + } = {}, ) { - this.kernel.traceEnabled = opts.debug ? true : false; + this.kernel.traceEnabled = opts.debug ?? false; + this.kernel.debugTimingEnabled = opts.debugTiming ?? false; } public run() { diff --git a/packages/@jsii/runtime/lib/program.ts b/packages/@jsii/runtime/lib/program.ts index d6844f0d71..41a35cc5f4 100644 --- a/packages/@jsii/runtime/lib/program.ts +++ b/packages/@jsii/runtime/lib/program.ts @@ -8,6 +8,7 @@ const version = packageInfo.version; const noStack = !!process.env.JSII_NOSTACK; const debug = !!process.env.JSII_DEBUG; +const debugTiming = !!process.env.JSII_DEBUG_TIMING; // This assumes FD#3 is opened for reading and writing. This is normally // performed by`bin/jsii-runtime.js`, and we will not be verifying this once @@ -24,7 +25,7 @@ const stdio = new SyncStdio({ }); const inout = new InputOutput(stdio); -const host = new KernelHost(inout, { debug, noStack }); +const host = new KernelHost(inout, { debug, noStack, debugTiming }); host.once('exit', process.exit.bind(process)); diff --git a/yarn.lock b/yarn.lock index cc6f43e014..f741542f52 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1971,6 +1971,11 @@ resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" integrity sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ== +"@types/lockfile@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@types/lockfile/-/lockfile-1.0.2.tgz#3f77e84171a2b7e3198bd5717c7547a54393baf8" + integrity sha512-jD5VbvhfMhaYN4M3qPJuhMVUg3Dfc4tvPvLEAXn6GXbs/ajDFtCQahX37GIE65ipTI3I+hEvNaXS3MYAn9Ce3Q== + "@types/minimatch@*", "@types/minimatch@^3.0.3": version "3.0.5" resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.5.tgz#1001cc5e6a3704b83c236027e77f2f58ea010f40" @@ -5582,6 +5587,13 @@ locate-path@^6.0.0: dependencies: p-locate "^5.0.0" +lockfile@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/lockfile/-/lockfile-1.0.4.tgz#07f819d25ae48f87e538e6578b6964a4981a5609" + integrity sha512-cvbTwETRfsFh4nHsL1eGWapU1XFi5Ot9E85sWAwia7Y7EgB7vfqcZhTKZ+l7hCGxSPoushMv5GKhT5PdLv03WA== + dependencies: + signal-exit "^3.0.2" + lodash.ismatch@^4.4.0: version "4.4.0" resolved "https://registry.yarnpkg.com/lodash.ismatch/-/lodash.ismatch-4.4.0.tgz#756cb5150ca3ba6f11085a78849645f188f85f37" From cb479e6312943bfd1e96c00044324ba2b978f6b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=A7=91=F0=9F=8F=BB=E2=80=8D=F0=9F=92=BB=20Romain=20M?= =?UTF-8?q?arcadier?= Date: Fri, 26 Aug 2022 18:22:51 +0200 Subject: [PATCH 2/6] Add v8-compile-cache to jsii runtime --- packages/@jsii/kernel/package.json | 3 ++- packages/@jsii/kernel/src/index.ts | 3 +++ yarn.lock | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/@jsii/kernel/package.json b/packages/@jsii/kernel/package.json index 76e82ed9a8..ccdb952360 100644 --- a/packages/@jsii/kernel/package.json +++ b/packages/@jsii/kernel/package.json @@ -34,7 +34,8 @@ "@jsii/spec": "^0.0.0", "fs-extra": "^10.1.0", "lockfile": "^1.0.4", - "tar": "^6.1.11" + "tar": "^6.1.11", + "v8-compile-cache": "^2.3.0" }, "devDependencies": { "@scope/jsii-calc-base": "^0.0.0", diff --git a/packages/@jsii/kernel/src/index.ts b/packages/@jsii/kernel/src/index.ts index c0b1652193..60166f09a6 100644 --- a/packages/@jsii/kernel/src/index.ts +++ b/packages/@jsii/kernel/src/index.ts @@ -1,3 +1,6 @@ +// eslint-disable-next-line @typescript-eslint/no-require-imports +require('v8-compile-cache'); + export * from './kernel'; import * as api from './api'; diff --git a/yarn.lock b/yarn.lock index f741542f52..c7ac2999fb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7875,7 +7875,7 @@ v8-compile-cache-lib@^3.0.1: resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf" integrity sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg== -v8-compile-cache@2.3.0, v8-compile-cache@^2.0.3: +v8-compile-cache@2.3.0, v8-compile-cache@^2.0.3, v8-compile-cache@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz#2de19618c66dc247dcfb6f99338035d8245a2cee" integrity sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA== From 22e3e1ad357e1ba3e0a0cb5502c4b32d22387fa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=A7=91=F0=9F=8F=BB=E2=80=8D=F0=9F=92=BB=20Romain=20M?= =?UTF-8?q?arcadier?= Date: Mon, 29 Aug 2022 15:09:03 +0200 Subject: [PATCH 3/6] re-organize code a bit, undo v8-code-cache (does not help) --- packages/@jsii/kernel/package.json | 3 +- .../{tar-cache => disk-cache}/digest-file.ts | 0 .../{tar-cache => disk-cache}/disk-cache.ts | 54 +++++++++++++++++-- packages/@jsii/kernel/src/disk-cache/index.ts | 1 + packages/@jsii/kernel/src/index.ts | 3 -- packages/@jsii/kernel/src/kernel.test.ts | 2 +- .../src/tar-cache/default-cache-root.ts | 1 + packages/@jsii/kernel/src/tar-cache/index.ts | 2 +- yarn.lock | 2 +- 9 files changed, 55 insertions(+), 13 deletions(-) rename packages/@jsii/kernel/src/{tar-cache => disk-cache}/digest-file.ts (100%) rename packages/@jsii/kernel/src/{tar-cache => disk-cache}/disk-cache.ts (78%) create mode 100644 packages/@jsii/kernel/src/disk-cache/index.ts diff --git a/packages/@jsii/kernel/package.json b/packages/@jsii/kernel/package.json index ccdb952360..76e82ed9a8 100644 --- a/packages/@jsii/kernel/package.json +++ b/packages/@jsii/kernel/package.json @@ -34,8 +34,7 @@ "@jsii/spec": "^0.0.0", "fs-extra": "^10.1.0", "lockfile": "^1.0.4", - "tar": "^6.1.11", - "v8-compile-cache": "^2.3.0" + "tar": "^6.1.11" }, "devDependencies": { "@scope/jsii-calc-base": "^0.0.0", diff --git a/packages/@jsii/kernel/src/tar-cache/digest-file.ts b/packages/@jsii/kernel/src/disk-cache/digest-file.ts similarity index 100% rename from packages/@jsii/kernel/src/tar-cache/digest-file.ts rename to packages/@jsii/kernel/src/disk-cache/digest-file.ts diff --git a/packages/@jsii/kernel/src/tar-cache/disk-cache.ts b/packages/@jsii/kernel/src/disk-cache/disk-cache.ts similarity index 78% rename from packages/@jsii/kernel/src/tar-cache/disk-cache.ts rename to packages/@jsii/kernel/src/disk-cache/disk-cache.ts index 04cfc4fc56..ece326ef9f 100644 --- a/packages/@jsii/kernel/src/tar-cache/disk-cache.ts +++ b/packages/@jsii/kernel/src/disk-cache/disk-cache.ts @@ -2,6 +2,7 @@ import { existsSync, mkdirSync, readdirSync, + readFileSync, realpathSync, rmdirSync, rmSync, @@ -48,19 +49,36 @@ export class DiskCache { process.once('beforeExit', () => this.pruneExpiredEntries()); } - public entryFor(path: string, ...comments: readonly string[]) { - const rawDigest = digestFile(path, ...comments); + public entry(...key: readonly string[]): Entry { + if (key.length === 0) { + throw new Error(`Cache entry key must contain at least 1 element!`); + } + return new Entry( join( this.#root, - ...comments.flatMap((s) => - s.replace(/[^@a-z0-9_.\\/-]+/g, '_').split(/[\\/]+/), + ...key.flatMap((s) => + s + .replace(/[^@a-z0-9_.\\/-]+/g, '_') + .split(/[\\/]+/) + .map((ss) => { + if (ss === '..') { + throw new Error( + `A cache entry key cannot contain a '..' path segment! (${s})`, + ); + } + return ss; + }), ), - rawDigest.toString('hex'), ), ); } + public entryFor(path: string, ...comments: readonly string[]): Entry { + const rawDigest = digestFile(path, ...comments); + return this.entry(...comments, rawDigest.toString('hex')); + } + public pruneExpiredEntries() { const cutOff = new Date(Date.now() - PRUNE_AFTER_MILLISECONDS); for (const entry of this.entries()) { @@ -148,6 +166,19 @@ export class Entry { } rmSync(this.path, { force: true, recursive: true }); }, + write: (name, content) => { + if (disposed) { + throw new Error( + `Cannot write ${join( + this.path, + name, + )} once the lock block as returned!`, + ); + } + + mkdirSync(dirname(join(this.path, name)), { recursive: true }); + writeFileSync(join(this.path, name), content); + }, touch: () => { if (disposed) { throw new Error( @@ -169,10 +200,23 @@ export class Entry { unlockSync(this.lockFile); } } + + public read(file: string): Buffer | undefined { + try { + return readFileSync(join(this.path, file)); + } catch (error: any) { + if (error.code === 'ENOENT') { + return undefined; + } + throw error; + } + } } export interface LockedEntry { delete(): void; + write(name: string, data: Buffer): void; + touch(): void; } diff --git a/packages/@jsii/kernel/src/disk-cache/index.ts b/packages/@jsii/kernel/src/disk-cache/index.ts new file mode 100644 index 0000000000..ad13a33657 --- /dev/null +++ b/packages/@jsii/kernel/src/disk-cache/index.ts @@ -0,0 +1 @@ +export * from './disk-cache'; diff --git a/packages/@jsii/kernel/src/index.ts b/packages/@jsii/kernel/src/index.ts index 60166f09a6..c0b1652193 100644 --- a/packages/@jsii/kernel/src/index.ts +++ b/packages/@jsii/kernel/src/index.ts @@ -1,6 +1,3 @@ -// eslint-disable-next-line @typescript-eslint/no-require-imports -require('v8-compile-cache'); - export * from './kernel'; import * as api from './api'; diff --git a/packages/@jsii/kernel/src/kernel.test.ts b/packages/@jsii/kernel/src/kernel.test.ts index 9365846744..c87bde929c 100644 --- a/packages/@jsii/kernel/src/kernel.test.ts +++ b/packages/@jsii/kernel/src/kernel.test.ts @@ -14,11 +14,11 @@ import { WireStruct, TOKEN_STRUCT, } from './api'; +import { DiskCache } from './disk-cache'; import { Kernel } from './kernel'; import { closeRecording, recordInteraction } from './recording'; import * as tar from './tar-cache'; import { defaultCacheRoot } from './tar-cache/default-cache-root'; -import { DiskCache } from './tar-cache/disk-cache'; /* eslint-disable require-atomic-updates */ diff --git a/packages/@jsii/kernel/src/tar-cache/default-cache-root.ts b/packages/@jsii/kernel/src/tar-cache/default-cache-root.ts index d61f57a404..6a86d5563b 100644 --- a/packages/@jsii/kernel/src/tar-cache/default-cache-root.ts +++ b/packages/@jsii/kernel/src/tar-cache/default-cache-root.ts @@ -10,6 +10,7 @@ export function defaultCacheRoot(): string { 'Library', 'Caches', 'com.amazonaws.jsii', + 'package-cache', ); break; case 'linux': diff --git a/packages/@jsii/kernel/src/tar-cache/index.ts b/packages/@jsii/kernel/src/tar-cache/index.ts index 4f68d82ad8..5bb6d3d9bc 100644 --- a/packages/@jsii/kernel/src/tar-cache/index.ts +++ b/packages/@jsii/kernel/src/tar-cache/index.ts @@ -3,8 +3,8 @@ import { tmpdir } from 'os'; import { join } from 'path'; import * as tar from 'tar'; +import { DiskCache } from '../disk-cache'; import { defaultCacheRoot } from './default-cache-root'; -import { DiskCache } from './disk-cache'; export type ExtractOptions = Omit< tar.ExtractOptions & tar.FileOptions, diff --git a/yarn.lock b/yarn.lock index c7ac2999fb..f741542f52 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7875,7 +7875,7 @@ v8-compile-cache-lib@^3.0.1: resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf" integrity sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg== -v8-compile-cache@2.3.0, v8-compile-cache@^2.0.3, v8-compile-cache@^2.3.0: +v8-compile-cache@2.3.0, v8-compile-cache@^2.0.3: version "2.3.0" resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz#2de19618c66dc247dcfb6f99338035d8245a2cee" integrity sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA== From 12eb3eddf265c2d173e37ad75afcf94667aab5a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=A7=91=F0=9F=8F=BB=E2=80=8D=F0=9F=92=BB=20Romain=20M?= =?UTF-8?q?arcadier?= Date: Tue, 30 Aug 2022 12:32:23 +0200 Subject: [PATCH 4/6] fix typos --- packages/@jsii/kernel/src/disk-cache/disk-cache.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/@jsii/kernel/src/disk-cache/disk-cache.ts b/packages/@jsii/kernel/src/disk-cache/disk-cache.ts index ece326ef9f..2e4c5d7047 100644 --- a/packages/@jsii/kernel/src/disk-cache/disk-cache.ts +++ b/packages/@jsii/kernel/src/disk-cache/disk-cache.ts @@ -161,7 +161,7 @@ export class Entry { delete: () => { if (disposed) { throw new Error( - `Cannot delete ${this.path} once the lock block as returned!`, + `Cannot delete ${this.path} once the lock block was returned!`, ); } rmSync(this.path, { force: true, recursive: true }); @@ -172,7 +172,7 @@ export class Entry { `Cannot write ${join( this.path, name, - )} once the lock block as returned!`, + )} once the lock block was returned!`, ); } @@ -182,7 +182,7 @@ export class Entry { touch: () => { if (disposed) { throw new Error( - `Cannot touch ${this.path} once the lock block as returned!`, + `Cannot touch ${this.path} once the lock block was returned!`, ); } if (this.pathExists) { From 6d61c11b206341775d6cdf59e9b70ba8114e5669 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=A7=91=F0=9F=8F=BB=E2=80=8D=F0=9F=92=BB=20Romain=20M?= =?UTF-8?q?arcadier?= Date: Tue, 30 Aug 2022 12:49:41 +0200 Subject: [PATCH 5/6] fix(pacmak): crash when generating java code A missing condition in the code generator could have resulted in attempting to read a property on undefined, resulting in a crash. --- packages/jsii-pacmak/lib/targets/java.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/java.ts b/packages/jsii-pacmak/lib/targets/java.ts index e328e5568c..79b5eb300e 100644 --- a/packages/jsii-pacmak/lib/targets/java.ts +++ b/packages/jsii-pacmak/lib/targets/java.ts @@ -1603,26 +1603,26 @@ class JavaGenerator extends Generator { ) { if (spec.isUnionTypeReference(type)) { validateTypeUnion.call(this, value, descr, type, parameterName); - } else { - const collectionType = type as spec.CollectionTypeReference; - if (collectionType.collection.kind === spec.CollectionKind.Array) { - validateArray.call( + } else if (spec.isCollectionTypeReference(type)) { + switch (type.collection.kind) { + case spec.CollectionKind.Array: + return validateArray.call( this, value, descr, - collectionType.collection.elementtype, + type.collection.elementtype, parameterName, isRawArray, ); - } else if (collectionType.collection.kind === spec.CollectionKind.Map) { - validateMap.call( + case spec.CollectionKind.Map: + return validateMap.call( this, value, descr, - collectionType.collection.elementtype, + type.collection.elementtype, parameterName, ); - } else { + default: throw new Error( `Unhandled collection kind: ${spec.describeTypeReference(type)}`, ); From aef0c02634d8e7c9fd07430c856c4600f92d39a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=F0=9F=A7=91=F0=9F=8F=BB=E2=80=8D=F0=9F=92=BB=20Romain=20M?= =?UTF-8?q?arcadier?= Date: Tue, 30 Aug 2022 12:49:41 +0200 Subject: [PATCH 6/6] fix(pacmak): crash when generating java code A missing condition in the code generator could have resulted in attempting to read a property on undefined, resulting in a crash. --- packages/jsii-pacmak/lib/targets/java.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/jsii-pacmak/lib/targets/java.ts b/packages/jsii-pacmak/lib/targets/java.ts index e328e5568c..79b5eb300e 100644 --- a/packages/jsii-pacmak/lib/targets/java.ts +++ b/packages/jsii-pacmak/lib/targets/java.ts @@ -1603,26 +1603,26 @@ class JavaGenerator extends Generator { ) { if (spec.isUnionTypeReference(type)) { validateTypeUnion.call(this, value, descr, type, parameterName); - } else { - const collectionType = type as spec.CollectionTypeReference; - if (collectionType.collection.kind === spec.CollectionKind.Array) { - validateArray.call( + } else if (spec.isCollectionTypeReference(type)) { + switch (type.collection.kind) { + case spec.CollectionKind.Array: + return validateArray.call( this, value, descr, - collectionType.collection.elementtype, + type.collection.elementtype, parameterName, isRawArray, ); - } else if (collectionType.collection.kind === spec.CollectionKind.Map) { - validateMap.call( + case spec.CollectionKind.Map: + return validateMap.call( this, value, descr, - collectionType.collection.elementtype, + type.collection.elementtype, parameterName, ); - } else { + default: throw new Error( `Unhandled collection kind: ${spec.describeTypeReference(type)}`, );