-
Notifications
You must be signed in to change notification settings - Fork 246
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(kernel): experimental runtime package cache
Adds an experimental (hence opt-in) feature that caches the contents of loaded libraries in a directory that persists between executions, in order to spare the time it takes to extract the tarballs. When this feature is enabled, packages present in the cache will be used as-is (i.e: they are not checked for tampering) instead of being extracted from the tarball. The cache is keyed on: - The hash of the tarball - The name of the library - The version of the library Objects in the cache will expire if they are not used for 30 days, and are subsequently removed from disk (this avoids a cache growing extremely large over time). In order to enable the feature, the following environment variables are used: - `JSII_RUNTIME_PACKAGE_CACHE` must be set to `enabled` in order for the package cache to be active at all; - `JSII_RUNTIME_PACKAGE_CACHE_ROOT` can be used to change which directory is used as a cache root. It defaults to: * On MacOS: `$HOME/Library/Caches/com.amazonaws.jsii` * On Linux: `$HOME/.cache/aws/jsii/package-cache` * On Windows: `%LOCALAPPDATA%\AWS\jsii\package-cache` * On other platforms: `$TMP/aws-jsii-package-cache` - `JSII_RUNTIME_PACKAGE_CACHE_TTL` can be used to change the default time entries will remain in cache before expiring if they are not used. This defaults to 30 days, and the value is expressed in days. Set to `0` to immediately expire all the cache's content. When troubleshooting load performance, it is possible to obtain timing data for some critical parts of the library load process within the jsii kernel by setting `JSII_DEBUG_TIMING` environment variable. Related to #3389
- Loading branch information
1 parent
93aec85
commit 5dc816e
Showing
12 changed files
with
528 additions
and
24 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
import { copyFileSync, linkSync, mkdirSync, readdirSync, statSync } from 'fs'; | ||
import { join } from 'path'; | ||
|
||
/** | ||
* Creates directories containing hard links if possible, and falls back on | ||
* copy otherwise. | ||
* | ||
* @param existing is the original file or directory to link. | ||
* @param destination is the nbew file or directory to create. | ||
*/ | ||
export function link(existing: string, destination: string): void { | ||
const stat = statSync(existing); | ||
if (!stat.isDirectory()) { | ||
try { | ||
linkSync(existing, destination); | ||
} catch { | ||
copyFileSync(existing, destination); | ||
} | ||
return; | ||
} | ||
|
||
mkdirSync(destination, { recursive: true }); | ||
for (const file of readdirSync(existing)) { | ||
link(join(existing, file), join(destination, file)); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
import { tmpdir } from 'os'; | ||
import { join } from 'path'; | ||
|
||
export function defaultCacheRoot(): string { | ||
switch (process.platform) { | ||
case 'darwin': | ||
if (process.env.HOME) | ||
return join( | ||
process.env.HOME, | ||
'Library', | ||
'Caches', | ||
'com.amazonaws.jsii', | ||
); | ||
break; | ||
case 'linux': | ||
if (process.env.HOME) | ||
return join(process.env.HOME, '.cache', 'aws', 'jsii', 'package-cache'); | ||
break; | ||
case 'win32': | ||
if (process.env.LOCALAPPDATA) | ||
return join(process.env.LOCALAPPDATA, 'AWS', 'jsii', 'package-cache'); | ||
break; | ||
default: | ||
// Fall back on putting in tmpdir() | ||
} | ||
return join(tmpdir(), 'aws-jsii-package-cache'); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
import { createHash } from 'crypto'; | ||
import { openSync, readSync, closeSync } from 'fs'; | ||
|
||
const ALGORITHM = 'sha256'; | ||
|
||
export function digestFile( | ||
path: string, | ||
...comments: readonly string[] | ||
): Buffer { | ||
const hash = createHash(ALGORITHM); | ||
|
||
const buffer = Buffer.alloc(16_384); | ||
const fd = openSync(path, 'r'); | ||
try { | ||
let bytesRead = 0; | ||
while ((bytesRead = readSync(fd, buffer)) > 0) { | ||
hash.update(buffer.slice(0, bytesRead)); | ||
} | ||
for (const comment of comments) { | ||
hash.update('\0'); | ||
hash.update(comment); | ||
} | ||
return hash.digest(); | ||
} finally { | ||
closeSync(fd); | ||
} | ||
} |
Oops, something went wrong.