diff --git a/.pnp.cjs b/.pnp.cjs index ca8eaa51bf6a..588c8b86c800 100755 --- a/.pnp.cjs +++ b/.pnp.cjs @@ -166,6 +166,10 @@ const RAW_RUNTIME_STATE = "name": "@yarnpkg/libzip",\ "reference": "workspace:packages/yarnpkg-libzip"\ },\ + {\ + "name": "@yarnpkg/minizip",\ + "reference": "workspace:packages/yarnpkg-minizip"\ + },\ {\ "name": "@yarnpkg/nm",\ "reference": "workspace:packages/yarnpkg-nm"\ @@ -208,6 +212,7 @@ const RAW_RUNTIME_STATE = ["@yarnpkg/fslib", ["workspace:packages/yarnpkg-fslib"]],\ ["@yarnpkg/libui", ["virtual:8d898fef98e932beba43b4d6618011f697787e7fd2f52624eb7daef58d4ae47c2c0be4a00e4cbfbae536ebdaf948ef2fd37ad5cb2fa89c56ad66c9e7ff10f073#workspace:packages/yarnpkg-libui", "workspace:packages/yarnpkg-libui"]],\ ["@yarnpkg/libzip", ["virtual:b73ceab179a3b4f89c4a5be81bd0c20a80eda623489cb284f304cc8104dbb771916bbc246d0ba809faebd8459cb6554cf114954badb021279ea7aee216456122#workspace:packages/yarnpkg-libzip", "workspace:packages/yarnpkg-libzip"]],\ + ["@yarnpkg/minizip", ["virtual:16110bda3ce959c103b1979c5d750ceb8ac9cfbd2049c118b6278e46e65aa65fd17e71e04a0ce5f75b7ca3203efd8e9c9b03c948a76c7f4bca807539915b5cfc#workspace:packages/yarnpkg-minizip", "workspace:packages/yarnpkg-minizip"]],\ ["@yarnpkg/monorepo", ["workspace:."]],\ ["@yarnpkg/nm", ["workspace:packages/yarnpkg-nm"]],\ ["@yarnpkg/parsers", ["workspace:packages/yarnpkg-parsers"]],\ @@ -264,6 +269,7 @@ const RAW_RUNTIME_STATE = ["@yarnpkg/eslint-config", "virtual:e470d99b1e4fdf4c5db5d090ff5472cdeba0404b7ffd31cd2efab3493dd184c67bc45f60c2ef1c040e2c41afe38c6280bffc5df2fbe3aefaa2b6eacf685ab07c#workspace:packages/eslint-config"],\ ["@yarnpkg/fslib", "workspace:packages/yarnpkg-fslib"],\ ["@yarnpkg/libzip", "virtual:b73ceab179a3b4f89c4a5be81bd0c20a80eda623489cb284f304cc8104dbb771916bbc246d0ba809faebd8459cb6554cf114954badb021279ea7aee216456122#workspace:packages/yarnpkg-libzip"],\ + ["@yarnpkg/minizip", "virtual:16110bda3ce959c103b1979c5d750ceb8ac9cfbd2049c118b6278e46e65aa65fd17e71e04a0ce5f75b7ca3203efd8e9c9b03c948a76c7f4bca807539915b5cfc#workspace:packages/yarnpkg-minizip"],\ ["@yarnpkg/sdks", "workspace:packages/yarnpkg-sdks"],\ ["@yarnpkg/types", "workspace:packages/yarnpkg-types"],\ ["chalk", "npm:3.0.0"],\ @@ -9589,6 +9595,35 @@ const RAW_RUNTIME_STATE = "linkType": "SOFT"\ }]\ ]],\ + ["@yarnpkg/minizip", [\ + ["virtual:16110bda3ce959c103b1979c5d750ceb8ac9cfbd2049c118b6278e46e65aa65fd17e71e04a0ce5f75b7ca3203efd8e9c9b03c948a76c7f4bca807539915b5cfc#workspace:packages/yarnpkg-minizip", {\ + "packageLocation": "./.yarn/__virtual__/@yarnpkg-minizip-virtual-e90355b441/1/packages/yarnpkg-minizip/",\ + "packageDependencies": [\ + ["@yarnpkg/minizip", "virtual:16110bda3ce959c103b1979c5d750ceb8ac9cfbd2049c118b6278e46e65aa65fd17e71e04a0ce5f75b7ca3203efd8e9c9b03c948a76c7f4bca807539915b5cfc#workspace:packages/yarnpkg-minizip"],\ + ["@types/prettier", "npm:1.19.0"],\ + ["@types/yarnpkg__fslib", null],\ + ["@yarnpkg/fslib", "workspace:packages/yarnpkg-fslib"],\ + ["prettier", "npm:1.19.1"],\ + ["tslib", "npm:2.6.2"]\ + ],\ + "packagePeers": [\ + "@types/yarnpkg__fslib",\ + "@yarnpkg/fslib"\ + ],\ + "linkType": "SOFT"\ + }],\ + ["workspace:packages/yarnpkg-minizip", {\ + "packageLocation": "./packages/yarnpkg-minizip/",\ + "packageDependencies": [\ + ["@yarnpkg/minizip", "workspace:packages/yarnpkg-minizip"],\ + ["@types/prettier", "npm:1.19.0"],\ + ["@yarnpkg/fslib", "workspace:packages/yarnpkg-fslib"],\ + ["prettier", "npm:1.19.1"],\ + ["tslib", "npm:2.6.2"]\ + ],\ + "linkType": "SOFT"\ + }]\ + ]],\ ["@yarnpkg/monorepo", [\ ["workspace:.", {\ "packageLocation": "./",\ @@ -9606,6 +9641,7 @@ const RAW_RUNTIME_STATE = ["@yarnpkg/eslint-config", "virtual:e470d99b1e4fdf4c5db5d090ff5472cdeba0404b7ffd31cd2efab3493dd184c67bc45f60c2ef1c040e2c41afe38c6280bffc5df2fbe3aefaa2b6eacf685ab07c#workspace:packages/eslint-config"],\ ["@yarnpkg/fslib", "workspace:packages/yarnpkg-fslib"],\ ["@yarnpkg/libzip", "virtual:b73ceab179a3b4f89c4a5be81bd0c20a80eda623489cb284f304cc8104dbb771916bbc246d0ba809faebd8459cb6554cf114954badb021279ea7aee216456122#workspace:packages/yarnpkg-libzip"],\ + ["@yarnpkg/minizip", "virtual:16110bda3ce959c103b1979c5d750ceb8ac9cfbd2049c118b6278e46e65aa65fd17e71e04a0ce5f75b7ca3203efd8e9c9b03c948a76c7f4bca807539915b5cfc#workspace:packages/yarnpkg-minizip"],\ ["@yarnpkg/sdks", "workspace:packages/yarnpkg-sdks"],\ ["@yarnpkg/types", "workspace:packages/yarnpkg-types"],\ ["chalk", "npm:3.0.0"],\ @@ -18953,6 +18989,7 @@ const RAW_RUNTIME_STATE = ["@types/semver", "npm:7.5.8"],\ ["@yarnpkg/fslib", "workspace:packages/yarnpkg-fslib"],\ ["@yarnpkg/libzip", "virtual:b73ceab179a3b4f89c4a5be81bd0c20a80eda623489cb284f304cc8104dbb771916bbc246d0ba809faebd8459cb6554cf114954badb021279ea7aee216456122#workspace:packages/yarnpkg-libzip"],\ + ["@yarnpkg/minizip", "virtual:16110bda3ce959c103b1979c5d750ceb8ac9cfbd2049c118b6278e46e65aa65fd17e71e04a0ce5f75b7ca3203efd8e9c9b03c948a76c7f4bca807539915b5cfc#workspace:packages/yarnpkg-minizip"],\ ["arg", "npm:5.0.2"],\ ["esbuild", [\ "esbuild-wasm",\ diff --git a/.yarn/cache/enhanced-resolve-npm-5.18.0-afcf74b9eb-e88463ef97.zip b/.yarn/cache/enhanced-resolve-npm-5.18.0-afcf74b9eb-e88463ef97.zip new file mode 100644 index 000000000000..5a4b42bc38fc Binary files /dev/null and b/.yarn/cache/enhanced-resolve-npm-5.18.0-afcf74b9eb-e88463ef97.zip differ diff --git a/bench/run-resolve.ts b/bench/run-resolve.ts new file mode 100644 index 000000000000..82995c79f07f --- /dev/null +++ b/bench/run-resolve.ts @@ -0,0 +1,36 @@ + +import * as fs from 'fs' +import { performance } from 'perf_hooks'; +import { ZipFS } from '@yarnpkg/libzip'; +import { MiniZipFS } from '@yarnpkg/minizip'; +import { PortablePath } from '@yarnpkg/fslib'; +import {createRequire} from 'module' +import path from 'path' +import {ResolverFactory} from 'enhanced-resolve' + + + +const formatMemoryUsage = (data: number) => `${Math.round(data / 1024 / 1024 * 100) / 100} MB`; + +const from = require.resolve('micromatch'); +// const baseFs= new ZipFS(from as PortablePath) +// const resolver = ResolverFactory.createResolver({fileSystem: {readFile: baseFs.readFileSync.bind(baseFs)}}) +console.log(from) +const req = createRequire(path.dirname(from)) + +globalThis.Error = class { + constructor() {} + static captureStackTrace = () => {} +} + + +const was = performance.now() +for (let i = 0; i < 50_000; i++) { + // resolver.resolveSync({}, from, `./fi${i}`) + try { + req.resolve(`./file${i}`) + } catch { + + } +} +console.log(performance.now() - was) diff --git a/bench/run.ts b/bench/run.ts new file mode 100644 index 000000000000..0b71c6c910cd --- /dev/null +++ b/bench/run.ts @@ -0,0 +1,80 @@ + +import * as fs from 'fs' +import { performance } from 'perf_hooks'; +import { ZipFS } from '@yarnpkg/libzip'; +import { MiniZipFS } from '@yarnpkg/minizip'; +import { PortablePath } from '@yarnpkg/fslib'; + + + + + +const cwd = '/Users/vadymh/.yarn/berry/cache'; +const files = fs.readdirSync(cwd).filter(f => f.endsWith('.zip')) + +let totalSize = 0 +for (const f of files) { + const file = `${cwd}/${f}`; + const stats = fs.statSync(file) + totalSize += stats.size +} + + +const formatMemoryUsage = (data: number) => `${Math.round(data / 1024 / 1024 * 100) / 100} MB`; + +console.log('files count: ' + files.length, 'size', formatMemoryUsage(totalSize)) + +const mem = process.argv.includes('--mem') + + +for (const zipper of [MiniZipFS, ZipFS]) { + for (let i = 0; i < 3; i++) { + let allFiles = 0 + const was = performance.now() + const memArr = [] + for (const f of files) { + const file = `${cwd}/${f}`; + + let fi = new zipper(file as PortablePath) + allFiles += fi.getAllFiles().length + if (mem){ + const memoryData = process.memoryUsage(); + memArr.push(memoryData.rss) + } + + fi.discardAndClose() + } + + // const memoryUsage = { + // rss: `${formatMemoryUsage(memoryData.rss)} -> Resident Set Size - total memory allocated for the process execution`, + // heapTotal: `${formatMemoryUsage(memoryData.heapTotal)} -> total size of the allocated heap`, + // heapUsed: `${formatMemoryUsage(memoryData.heapUsed)} -> actual memory used during the execution`, + // external: `${formatMemoryUsage(memoryData.external)} -> V8 external memory`, + // }; + + console.log(`${zipper.name}: try ${i + 1}`) + + if (mem) { + memArr.sort((a, b) => a-b) + console.log({ + p50: formatMemoryUsage(memArr[Math.floor(memArr.length / 2)]), + p90: formatMemoryUsage(memArr[Math.floor(memArr.length * 0.9)]), + p95: formatMemoryUsage(memArr[Math.floor(memArr.length * 0.95)]), + p99: formatMemoryUsage(memArr[Math.floor(memArr.length * 0.99)]), + }) + } else { + const took = performance.now() - was + console.log({ + took, + allFiles + }) + } + + // console.log(memoryUsage) + } +} + + + + + diff --git a/package.json b/package.json index 8ae492bda1f5..becd3e94a349 100644 --- a/package.json +++ b/package.json @@ -17,8 +17,10 @@ "@yarnpkg/eslint-config": "workspace:^", "@yarnpkg/fslib": "workspace:^", "@yarnpkg/libzip": "workspace:^", + "@yarnpkg/minizip": "workspace:^", "@yarnpkg/sdks": "workspace:^", "clipanion": "^4.0.0-rc.2", + "enhanced-resolve": "^5.18.0", "esbuild": "npm:esbuild-wasm@^0.23.0", "eslint": "^8.57.0", "jest": "^29.2.1", diff --git a/packages/plugin-pnp/sources/PnpLinker.ts b/packages/plugin-pnp/sources/PnpLinker.ts index d19646d6adef..71f2a72a53c6 100644 --- a/packages/plugin-pnp/sources/PnpLinker.ts +++ b/packages/plugin-pnp/sources/PnpLinker.ts @@ -285,6 +285,7 @@ export class PnpInstaller implements Installer { fallbackExclusionList, fallbackPool, ignorePattern, + zipImplementation: this.opts.project.configuration.get(`minizip`) ? `minizip` : `libzip`, packageRegistry, shebang, }); diff --git a/packages/plugin-pnp/sources/index.ts b/packages/plugin-pnp/sources/index.ts index 857485dc5133..2c2c6a87a963 100644 --- a/packages/plugin-pnp/sources/index.ts +++ b/packages/plugin-pnp/sources/index.ts @@ -70,6 +70,7 @@ declare module '@yarnpkg/core' { nodeLinker: string; winLinkType: string; pnpMode: string; + minizip: boolean pnpShebang: string; pnpIgnorePatterns: Array; pnpEnableEsmLoader: boolean; @@ -90,6 +91,11 @@ const plugin: Plugin = { type: SettingsType.STRING, default: `pnp`, }, + minizip: { + description: `Whether Yarn should use minizip to extract archives`, + type: SettingsType.BOOLEAN, + default: false + }, winLinkType: { description: `Whether Yarn should use Windows Junctions or symlinks when creating links on Windows.`, type: SettingsType.STRING, diff --git a/packages/yarnpkg-minizip/.babelrc.js b/packages/yarnpkg-minizip/.babelrc.js new file mode 100644 index 000000000000..23d1a822c0d7 --- /dev/null +++ b/packages/yarnpkg-minizip/.babelrc.js @@ -0,0 +1,5 @@ +module.exports = { + ignore: [ + `./sources/libzip.js`, + ], +}; diff --git a/packages/yarnpkg-minizip/README.md b/packages/yarnpkg-minizip/README.md new file mode 100644 index 000000000000..ebc59b9251b9 --- /dev/null +++ b/packages/yarnpkg-minizip/README.md @@ -0,0 +1,3 @@ +# `@yarnpkg/minizip` + +Lightweight, fast, readonly zip fs for pnp runtime. \ No newline at end of file diff --git a/packages/yarnpkg-minizip/package.json b/packages/yarnpkg-minizip/package.json new file mode 100644 index 000000000000..a3da329b84f6 --- /dev/null +++ b/packages/yarnpkg-minizip/package.json @@ -0,0 +1,50 @@ +{ + "name": "@yarnpkg/minizip", + "version": "3.1.0", + "license": "BSD-2-Clause", + "main": "./sources/index.ts", + "exports": { + ".": { + "default": "./sources/index.ts" + }, + "./package.json": "./package.json" + }, + "scripts": { + "postpack": "rm -rf lib", + "prepack": "run build:compile \"$(pwd)\"", + "release": "yarn npm publish" + }, + "publishConfig": { + "main": "./lib/index.js", + "browser": "./lib/index.js", + "exports": { + ".": { + "default": "./lib/index.js" + }, + "./package.json": "./package.json" + } + }, + "files": [ + "/lib/**/*" + ], + "repository": { + "type": "git", + "url": "ssh://git@github.com/yarnpkg/berry.git", + "directory": "packages/yarnpkg-minizip" + }, + "devDependencies": { + "@types/prettier": "1.19.0", + "prettier": "^1.19.1" + }, + "dependencies": { + "@yarnpkg/fslib": "workspace:^", + "tslib": "^2.4.0" + }, + "peerDependencies": { + "@yarnpkg/fslib": "workspace:^" + }, + "engines": { + "node": ">=18.12.0" + }, + "stableVersion": "3.1.0" +} diff --git a/packages/yarnpkg-minizip/sources/MiniZipFS.ts b/packages/yarnpkg-minizip/sources/MiniZipFS.ts new file mode 100644 index 000000000000..f693e084a9d3 --- /dev/null +++ b/packages/yarnpkg-minizip/sources/MiniZipFS.ts @@ -0,0 +1,1151 @@ +import { Dirent, DirentNoPath, ReaddirOptions } from '@yarnpkg/fslib'; +import { WatchOptions, WatchCallback, Watcher, Dir, Stats, BigIntStats, StatSyncOptions, StatOptions } from '@yarnpkg/fslib'; +import { FakeFS, MkdirOptions, RmdirOptions, RmOptions, WriteFileOptions, OpendirOptions } from '@yarnpkg/fslib'; +import { CreateReadStreamOptions, CreateWriteStreamOptions, BasePortableFakeFS, ExtractHintOptions, WatchFileCallback, WatchFileOptions, StatWatcher } from '@yarnpkg/fslib'; +import { NodeFS } from '@yarnpkg/fslib'; +import { opendir } from '@yarnpkg/fslib'; +import { watchFile, unwatchFile, unwatchAllFiles } from '@yarnpkg/fslib'; +import { errors, statUtils } from '@yarnpkg/fslib'; +import { FSPath, PortablePath, ppath, Filename } from '@yarnpkg/fslib'; +import { ReadStream, WriteStream, constants } from 'fs'; +import { PassThrough } from 'stream'; +import zlib from 'zlib'; + + +const UNIX = 3 + +export type ZipPathOptions = { + baseFs?: FakeFS; + +}; + +export interface Entry { + name: string; + compressionMethod: number; + size: number; + os: number; + isSymbolicLink: boolean; + crc: number //needed? + compressedSize: number; + externalAttributes: number; + mTime: number + fileContentOffset: number; + index: number; +} + + +const SIGNATURE = { + CENTRAL_DIRECTORY: 0x02014b50, + END_OF_CENTRAL_DIRECTORY: 0x06054b50, +}; + +const noCommentCDSize = 22; + +// const error = () => { throw new Error('not supported algo'); }; + +// type Decompress = (buf: Buffer) => Buffer; + +// const COMPRESSION_METHODS: Record = { +// 0: null, // 'STORED' +// 1: error, // 'SHRUNK' +// // 8: null, +// 8: buf => zlib.inflateRawSync(buf), // 'DEFLATED' +// 9: error, // 'DEFLATE64' +// 12: error, // 'BZIP2' +// 14: error, // 'LZMA' +// 19: error, // 'LZ77' +// 93: error, // 'ZSTD' +// 97: error, // 'XZ +// }; + + + +// function readFile(baseFs: BasePortableFakeFS, fd: number, entry: Entry) { +// const contentBuffer = Buffer.alloc(entry.compressedSize); +// baseFs.readSync(fd, contentBuffer, 0, entry.compressedSize, entry.fileContentOffset); +// const decompress = COMPRESSION_METHODS[entry.compressionMethod]; +// if (decompress === null) { +// return contentBuffer; +// } +// return decompress(contentBuffer); +// } + +function readZipSync(baseFs: BasePortableFakeFS, fd: number): Entry[] { + + const stats = baseFs.fstatSync(fd); + const fileSize = stats.size; + + if (fileSize < noCommentCDSize) { + throw new Error('Invalid ZIP file: EOCD not found'); + } + + let eocdOffset = -1; + + // fast read if no comment + let cdBuffer = Buffer.alloc(noCommentCDSize); + baseFs.readSync( + fd, + cdBuffer, + 0, + noCommentCDSize, + fileSize - noCommentCDSize + ); + + if (cdBuffer.readUInt32LE(0) === SIGNATURE.END_OF_CENTRAL_DIRECTORY) { + eocdOffset = 0 + } else { + const bufferSize = Math.min(65557, fileSize); + cdBuffer = Buffer.alloc(bufferSize); + + // Read potential EOCD area + baseFs.readSync( + fd, + cdBuffer, + 0, + bufferSize, + Math.max(0, fileSize - bufferSize) + ); + + // Find EOCD signature + for (let i = cdBuffer.length - 4; i >= 0; i--) { + if (cdBuffer.readUInt32LE(i) === SIGNATURE.END_OF_CENTRAL_DIRECTORY) { + eocdOffset = i; + break; + } + } + if (eocdOffset === -1) throw new Error('Invalid ZIP file: EOCD not found'); + } + + + + const totalEntries = cdBuffer.readUInt16LE(eocdOffset + 10); + const centralDirSize = cdBuffer.readUInt32LE(eocdOffset + 12); + const centralDirOffset = cdBuffer.readUInt32LE(eocdOffset + 16); + + // Read central directory + const centralDirBuffer = Buffer.alloc(centralDirSize); + baseFs.readSync(fd, centralDirBuffer, 0, centralDirBuffer.length, centralDirOffset); + + const entries: Entry[] = []; + let offset = 0; + let index = 0 + while (offset < centralDirBuffer.length && index < totalEntries) { // rm offset < centralDirBuffer.length? + if (centralDirBuffer.readUInt32LE(offset) !== SIGNATURE.CENTRAL_DIRECTORY) break; + const versionMadeBy = centralDirBuffer.readUInt16LE(offset + 4); + const os = versionMadeBy >>> 8; + const compressionMethod = centralDirBuffer.readUInt16LE(offset + 10); + const crc = centralDirBuffer.readUInt32LE(offset + 16); + const nameLength = centralDirBuffer.readUInt16LE(offset + 28); + const extraLength = centralDirBuffer.readUInt16LE(offset + 30); + const commentLength = centralDirBuffer.readUInt16LE(offset + 32); + const localHeaderOffset = centralDirBuffer.readUInt32LE(offset + 42); + const name = centralDirBuffer.toString('utf8', offset + 46, offset + 46 + nameLength); + const fileContentOffset = localHeaderOffset + 30 + nameLength + extraLength + const externalAttributes = centralDirBuffer.readUInt32LE(offset + 38); + + entries.push({ + index, + name, + os, + mTime: 0, //we dont care, + crc, //needed? + compressionMethod, + isSymbolicLink: os === UNIX && ((externalAttributes >>> 16) & constants.S_IFMT) === constants.S_IFLNK, + size: centralDirBuffer.readUInt32LE(offset + 24), + compressedSize: centralDirBuffer.readUInt32LE(offset + 20), + externalAttributes, + fileContentOffset, + }); + + index += 1 + offset += 46 + nameLength + extraLength + commentLength; + } + + return entries; + +} + + + +export class MiniZipFS extends BasePortableFakeFS { + + + private readonly baseFs!: FakeFS + private readonly path: PortablePath | null; + + private readonly stats: Stats; + + private readonly listings: Map> = new Map(); + private readonly entries: Map = new Map(); + + /** + * A cache of indices mapped to file sources. + * Populated by `setFileSource` calls. + * Required for supporting read after write. + */ + // private readonly fileSources: Map = new Map(); + + private readonly fds: Map = new Map(); + private nextFd: number = 0; + private archiveFd: number; + private hasSymlinks: boolean; + + + constructor(p: PortablePath, opts: ZipPathOptions = {}) { + super(); + const { baseFs = new NodeFS() } = opts; + this.baseFs = baseFs; + this.path = p; + + this.stats = this.baseFs!.statSync(p); + + this.listings.set(PortablePath.root, new Set()); + + this.archiveFd = baseFs.openSync(p, 'r'); + this.hasSymlinks = false + for (const entry of readZipSync(this.baseFs, this.archiveFd)) { + const raw = entry.name as PortablePath; + if (ppath.isAbsolute(raw)) + continue; + + const p = ppath.resolve(PortablePath.root, raw); + this.registerEntry(p, entry); + if (entry.isSymbolicLink) { + this.hasSymlinks = true + } + + // If the raw path is a directory, register it + // to prevent empty folder being skipped + if (raw.endsWith(`/`)) { + this.registerListing(p); + } + } + } + + saveAndClose() { + this.clean() + } + discardAndClose() { + this.clean() + } + private clean() { + unwatchAllFiles(this); + this.baseFs.closeSync(this.archiveFd); + } + + getExtractHint(hints: ExtractHintOptions) { + for (const fileName of this.entries.keys()) { + const ext = this.pathUtils.extname(fileName); + if (hints.relevantExtensions.has(ext)) { + return true; + } + } + + return false; + } + + getRealPath() { + if (!this.path) + throw new Error(`ZipFS don't have real paths when loaded from a buffer`); + + return this.path; + } + + resolve(p: PortablePath) { + return ppath.resolve(PortablePath.root, p); + } + + async openPromise(p: PortablePath, flags: string, mode?: number) { + return this.openSync(p, flags, mode); + } + + openSync(p: PortablePath, flags: string, mode?: number) { + const fd = this.nextFd++; + this.fds.set(fd, { cursor: 0, p }); + return fd; + } + + hasOpenFileHandles(): boolean { + return !!this.fds.size; + } + + async opendirPromise(p: PortablePath, opts?: OpendirOptions) { + return this.opendirSync(p, opts); + } + + opendirSync(p: PortablePath, opts: OpendirOptions = {}): Dir { + const resolvedP = this.resolveFilename(`opendir '${p}'`, p); + if (!this.entries.has(resolvedP) && !this.listings.has(resolvedP)) + throw errors.ENOENT(`opendir '${p}'`); + + const directoryListing = this.listings.get(resolvedP); + if (!directoryListing) + throw errors.ENOTDIR(`opendir '${p}'`); + + const entries = [...directoryListing]; + + const fd = this.openSync(resolvedP, `r`); + + const onClose = () => { + this.closeSync(fd); + }; + + return opendir(this, resolvedP, entries, { onClose }); + } + + async readPromise(fd: number, buffer: Buffer, offset?: number, length?: number, position?: number | null) { + return this.readSync(fd, buffer, offset, length, position); + } + + readSync(fd: number, buffer: Buffer, offset: number = 0, length: number = buffer.byteLength, position: number | null = -1) { + const entry = this.fds.get(fd); + if (typeof entry === `undefined`) + throw errors.EBADF(`read`); + + const realPosition = position === -1 || position === null + ? entry.cursor + : position; + + const source = this.readFileSync(entry.p); + source.copy(buffer, offset, realPosition, realPosition + length); + + const bytesRead = Math.max(0, Math.min(source.length - realPosition, length)); + if (position === -1 || position === null) + entry.cursor += bytesRead; + + return bytesRead; + } + + writePromise(fd: number, buffer: Buffer, offset?: number, length?: number, position?: number): Promise; + writePromise(fd: number, buffer: string, position?: number): Promise; + async writePromise(fd: number, buffer: Buffer | string, offset?: number, length?: number, position?: number): Promise { + if (typeof buffer === `string`) { + return this.writeSync(fd, buffer, position); + } else { + return this.writeSync(fd, buffer, offset, length, position); + } + } + + writeSync(fd: number, buffer: Buffer, offset?: number, length?: number, position?: number): number; + writeSync(fd: number, buffer: string, position?: number): number; + writeSync(fd: number, buffer: Buffer | string, offset?: number, length?: number, position?: number): never { + const entry = this.fds.get(fd); + if (typeof entry === `undefined`) + throw errors.EBADF(`read`); + + throw new Error(`Unimplemented`); + } + + async closePromise(fd: number) { + return this.closeSync(fd); + } + + closeSync(fd: number) { + const entry = this.fds.get(fd); + if (typeof entry === `undefined`) + throw errors.EBADF(`read`); + + this.fds.delete(fd); + } + + createReadStream(p: PortablePath | null, { encoding }: CreateReadStreamOptions = {}): ReadStream { + if (p === null) + throw new Error(`Unimplemented`); + + const fd = this.openSync(p, `r`); + + const stream = Object.assign( + new PassThrough({ + emitClose: true, + autoDestroy: true, + destroy: (error, callback) => { + clearImmediate(immediate); + this.closeSync(fd); + callback(error); + }, + }), + { + close() { + stream.destroy(); + }, + bytesRead: 0, + path: p, + // "This property is `true` if the underlying file has not been opened yet" + pending: false, + }, + ); + + const immediate = setImmediate(async () => { + try { + const data = await this.readFilePromise(p, encoding); + stream.bytesRead = data.length; + stream.end(data); + } catch (error) { + stream.destroy(error); + } + }); + + return stream; + } + + createWriteStream(p: PortablePath | null, { encoding }: CreateWriteStreamOptions = {}): WriteStream { + throw errors.EROFS(`open '${p}'`); + } + + async realpathPromise(p: PortablePath) { + return this.realpathSync(p); + } + + realpathSync(p: PortablePath): PortablePath { + const resolvedP = this.resolveFilename(`lstat '${p}'`, p); + + if (!this.entries.has(resolvedP) && !this.listings.has(resolvedP)) + throw errors.ENOENT(`lstat '${p}'`); + + return resolvedP; + } + + async existsPromise(p: PortablePath) { + return this.existsSync(p); + } + + existsSync(p: PortablePath): boolean { + + if (!this.hasSymlinks) { + const resolvedP = ppath.resolve(PortablePath.root, p); + return this.entries.has(resolvedP) || this.listings.has(resolvedP); + } + + let resolvedP; + + try { + resolvedP = this.resolveFilename(`stat '${p}'`, p, undefined, false); + } catch (error) { + return false; + } + + if (resolvedP === undefined) + return false; + + return this.entries.has(resolvedP) || this.listings.has(resolvedP); + } + + async accessPromise(p: PortablePath, mode?: number) { + return this.accessSync(p, mode); + } + + accessSync(p: PortablePath, mode: number = constants.F_OK) { + const resolvedP = this.resolveFilename(`access '${p}'`, p); + + if (!this.entries.has(resolvedP) && !this.listings.has(resolvedP)) + throw errors.ENOENT(`access '${p}'`); + + if (mode & constants.W_OK) { + throw errors.EROFS(`access '${p}'`); + } + } + + // https://github.com/DefinitelyTyped/DefinitelyTyped/blob/51d793492d4c2e372b01257668dcd3afc58d7352/types/node/v16/fs.d.ts#L1042-L1059 + async statPromise(p: PortablePath): Promise; + async statPromise(p: PortablePath, opts: (StatOptions & { bigint?: false | undefined }) | undefined): Promise; + async statPromise(p: PortablePath, opts: StatOptions & { bigint: true }): Promise; + async statPromise(p: PortablePath, opts?: StatOptions): Promise; + async statPromise(p: PortablePath, opts: StatOptions = { bigint: false }): Promise { + if (opts.bigint) + return this.statSync(p, { bigint: true }); + + return this.statSync(p); + } + + // https://github.com/DefinitelyTyped/DefinitelyTyped/blob/51d793492d4c2e372b01257668dcd3afc58d7352/types/node/v16/fs.d.ts#L931-L967 + statSync(p: PortablePath): Stats; + statSync(p: PortablePath, opts?: StatSyncOptions & { bigint?: false | undefined, throwIfNoEntry: false }): Stats | undefined; + statSync(p: PortablePath, opts: StatSyncOptions & { bigint: true, throwIfNoEntry: false }): BigIntStats | undefined; + statSync(p: PortablePath, opts?: StatSyncOptions & { bigint?: false | undefined }): Stats; + statSync(p: PortablePath, opts: StatSyncOptions & { bigint: true }): BigIntStats; + statSync(p: PortablePath, opts: StatSyncOptions & { bigint: boolean, throwIfNoEntry?: false | undefined }): Stats | BigIntStats; + statSync(p: PortablePath, opts?: StatSyncOptions): Stats | BigIntStats | undefined; + statSync(p: PortablePath, opts: StatSyncOptions = { bigint: false, throwIfNoEntry: true }): Stats | BigIntStats | undefined { + const resolvedP = this.resolveFilename(`stat '${p}'`, p, undefined, opts.throwIfNoEntry); + if (resolvedP === undefined) + return undefined; + + if (!this.entries.has(resolvedP) && !this.listings.has(resolvedP)) { + if (opts.throwIfNoEntry === false) + return undefined; + + throw errors.ENOENT(`stat '${p}'`); + } + + if (p[p.length - 1] === `/` && !this.listings.has(resolvedP)) + throw errors.ENOTDIR(`stat '${p}'`); + + return this.statImpl(`stat '${p}'`, resolvedP, opts); + } + + async fstatPromise(fd: number): Promise; + async fstatPromise(fd: number, opts: { bigint: true }): Promise; + async fstatPromise(fd: number, opts?: { bigint: boolean }): Promise; + async fstatPromise(fd: number, opts?: { bigint: boolean }) { + return this.fstatSync(fd, opts); + } + + fstatSync(fd: number): Stats; + fstatSync(fd: number, opts: { bigint: true }): BigIntStats; + fstatSync(fd: number, opts?: { bigint: boolean }): BigIntStats | Stats; + fstatSync(fd: number, opts?: { bigint: boolean }) { + const entry = this.fds.get(fd); + if (typeof entry === `undefined`) + throw errors.EBADF(`fstatSync`); + + const { p } = entry; + + const resolvedP = this.resolveFilename(`stat '${p}'`, p); + + if (!this.entries.has(resolvedP) && !this.listings.has(resolvedP)) + throw errors.ENOENT(`stat '${p}'`); + + if (p[p.length - 1] === `/` && !this.listings.has(resolvedP)) + throw errors.ENOTDIR(`stat '${p}'`); + + return this.statImpl(`fstat '${p}'`, resolvedP, opts); + } + + // https://github.com/DefinitelyTyped/DefinitelyTyped/blob/51d793492d4c2e372b01257668dcd3afc58d7352/types/node/v16/fs.d.ts#L1042-L1059 + async lstatPromise(p: PortablePath): Promise; + async lstatPromise(p: PortablePath, opts: (StatOptions & { bigint?: false | undefined }) | undefined): Promise; + async lstatPromise(p: PortablePath, opts: StatOptions & { bigint: true }): Promise; + async lstatPromise(p: PortablePath, opts?: StatOptions): Promise; + async lstatPromise(p: PortablePath, opts: StatOptions = { bigint: false }): Promise { + if (opts.bigint) + return this.lstatSync(p, { bigint: true }); + + return this.lstatSync(p); + } + + // https://github.com/DefinitelyTyped/DefinitelyTyped/blob/51d793492d4c2e372b01257668dcd3afc58d7352/types/node/v16/fs.d.ts#L931-L967 + lstatSync(p: PortablePath): Stats; + lstatSync(p: PortablePath, opts?: StatSyncOptions & { bigint?: false | undefined, throwIfNoEntry: false }): Stats | undefined; + lstatSync(p: PortablePath, opts: StatSyncOptions & { bigint: true, throwIfNoEntry: false }): BigIntStats | undefined; + lstatSync(p: PortablePath, opts?: StatSyncOptions & { bigint?: false | undefined }): Stats; + lstatSync(p: PortablePath, opts: StatSyncOptions & { bigint: true }): BigIntStats; + lstatSync(p: PortablePath, opts: StatSyncOptions & { bigint: boolean, throwIfNoEntry?: false | undefined }): Stats | BigIntStats; + lstatSync(p: PortablePath, opts?: StatSyncOptions): Stats | BigIntStats | undefined; + lstatSync(p: PortablePath, opts: StatSyncOptions = { bigint: false, throwIfNoEntry: true }): Stats | BigIntStats | undefined { + const resolvedP = this.resolveFilename(`lstat '${p}'`, p, false, opts.throwIfNoEntry); + if (resolvedP === undefined) + return undefined; + + if (!this.entries.has(resolvedP) && !this.listings.has(resolvedP)) { + if (opts.throwIfNoEntry === false) + return undefined; + + throw errors.ENOENT(`lstat '${p}'`); + } + + if (p[p.length - 1] === `/` && !this.listings.has(resolvedP)) + throw errors.ENOTDIR(`lstat '${p}'`); + + return this.statImpl(`lstat '${p}'`, resolvedP, opts); + } + + private statImpl(reason: string, p: PortablePath, opts: { bigint: true }): BigIntStats; + private statImpl(reason: string, p: PortablePath, opts?: { bigint?: false }): Stats; + private statImpl(reason: string, p: PortablePath, opts?: { bigint?: boolean }): Stats | BigIntStats; + private statImpl(reason: string, p: PortablePath, opts: { bigint?: boolean } = {}): Stats | BigIntStats { + const entry = this.entries.get(p); + + // File, or explicit directory + if (typeof entry !== `undefined`) { + + const uid = this.stats.uid; + const gid = this.stats.gid; + + const size = (entry.size >>> 0); + const blksize = 512; + const blocks = Math.ceil(size / blksize); + + const mtimeMs = (entry.mTime >>> 0) * 1000; + const atimeMs = mtimeMs; + const birthtimeMs = mtimeMs; + const ctimeMs = mtimeMs; + + const atime = new Date(atimeMs); + const birthtime = new Date(birthtimeMs); + const ctime = new Date(ctimeMs); + const mtime = new Date(mtimeMs); + + const type = this.listings.has(p) + ? constants.S_IFDIR + : this.isSymbolicLink(entry) + ? constants.S_IFLNK + : constants.S_IFREG; + + const defaultMode = type === constants.S_IFDIR + ? 0o755 + : 0o644; + + const mode = type | (this.getUnixMode(entry, defaultMode) & 0o777); + const crc = entry.crc + + const statInstance = Object.assign(new statUtils.StatEntry(), { uid, gid, size, blksize, blocks, atime, birthtime, ctime, mtime, atimeMs, birthtimeMs, ctimeMs, mtimeMs, mode, crc }); + return opts.bigint === true ? statUtils.convertToBigIntStats(statInstance) : statInstance; + } + + // Implicit directory + if (this.listings.has(p)) { + const uid = this.stats.uid; + const gid = this.stats.gid; + + const size = 0; + const blksize = 512; + const blocks = 0; + + const atimeMs = this.stats.mtimeMs; + const birthtimeMs = this.stats.mtimeMs; + const ctimeMs = this.stats.mtimeMs; + const mtimeMs = this.stats.mtimeMs; + + const atime = new Date(atimeMs); + const birthtime = new Date(birthtimeMs); + const ctime = new Date(ctimeMs); + const mtime = new Date(mtimeMs); + + const mode = constants.S_IFDIR | 0o755; + const crc = 0; + + const statInstance = Object.assign(new statUtils.StatEntry(), { uid, gid, size, blksize, blocks, atime, birthtime, ctime, mtime, atimeMs, birthtimeMs, ctimeMs, mtimeMs, mode, crc }); + return opts.bigint === true ? statUtils.convertToBigIntStats(statInstance) : statInstance; + } + + throw new Error(`Unreachable`); + } + + private getUnixMode(entry: Entry , defaultMode: number) { + // const rc = this.libzip.file.getExternalAttributes(this.zip, index, 0, 0, this.libzip.uint08S, this.libzip.uint32S); + + if (entry.os !== UNIX) + return defaultMode; + + return entry.externalAttributes >>> 16; + } + + private registerListing(p: PortablePath) { + const existingListing = this.listings.get(p); + if (existingListing) + return existingListing; + + const parentListing = this.registerListing(ppath.dirname(p)); + parentListing.add(ppath.basename(p)); + + const newListing = new Set(); + this.listings.set(p, newListing); + + return newListing; + } + + private registerEntry(p: PortablePath, entry: Entry) { + const parentListing = this.registerListing(ppath.dirname(p)); + parentListing.add(ppath.basename(p)); + + this.entries.set(p, entry); + } + + + + + private resolveFilename(reason: string, p: PortablePath, resolveLastComponent?: boolean): PortablePath; + private resolveFilename(reason: string, p: PortablePath, resolveLastComponent: boolean | undefined, throwIfNoEntry: boolean | undefined): PortablePath | undefined; + private resolveFilename(reason: string, p: PortablePath, resolveLastComponent: boolean = true, throwIfNoEntry = true): PortablePath | undefined { + let resolvedP = ppath.resolve(PortablePath.root, p); + if (resolvedP === `/`) + return PortablePath.root; + + const entry = this.entries.get(resolvedP); + if (resolveLastComponent && entry !== undefined) { + if (this.hasSymlinks && this.isSymbolicLink(entry)) { + const target = this.getFileSource(entry).toString() as PortablePath; + return this.resolveFilename(reason, ppath.resolve(ppath.dirname(resolvedP), target), true, throwIfNoEntry); + } else { + return resolvedP; + } + } + + while (true) { + const parentP = this.resolveFilename(reason, ppath.dirname(resolvedP), true, throwIfNoEntry); + if (parentP === undefined) + return parentP; + + const isDir = this.listings.has(parentP); + const doesExist = this.entries.has(parentP); + + if (!isDir && !doesExist) { + if (throwIfNoEntry === false) + return undefined; + + throw errors.ENOENT(reason); + } + if (!isDir) + throw errors.ENOTDIR(reason); + + resolvedP = ppath.resolve(parentP, ppath.basename(resolvedP)); + if (!resolveLastComponent || !this.hasSymlinks) + break; + + // I'm not sure this is correct + const entry = this.entries.get(resolvedP.slice(1) as PortablePath); + if (!entry) { + break + } + // const index = this.libzip.name.locate(this.zip, resolvedP.slice(1), 0); + // if (index === -1) + // break; + // + + if (this.isSymbolicLink(entry)) { + const target = this.getFileSource(entry).toString() as PortablePath; + resolvedP = ppath.resolve(ppath.dirname(resolvedP), target); + } else { + break; + } + } + + return resolvedP; + } + + + private isSymbolicLink(entry: Entry) { + return entry.isSymbolicLink + } + + private getFileSource(entry: Entry): Buffer; + private getFileSource(entry: Entry, opts: { asyncDecompress: false }): Buffer; + private getFileSource(entry: Entry, opts: { asyncDecompress: true }): Promise; + private getFileSource(entry: Entry, opts: { asyncDecompress: boolean }): Promise | Buffer; + private getFileSource(entry: Entry, opts: { asyncDecompress: boolean } = { asyncDecompress: false }): Promise | Buffer { + // const { index } = entry; + // const cachedFileSource = this.fileSources.get(index); //fileSourceCache?? + // if (typeof cachedFileSource !== `undefined`) + // return cachedFileSource; + + + const data = Buffer.alloc(entry.compressedSize); + this.baseFs.readSync(this.archiveFd, data, 0, entry.compressedSize, entry.fileContentOffset); + + if (entry.compressionMethod === 0) { + // this.fileSources.set(index, data); + return data; + } else if (opts.asyncDecompress) { + return new Promise((resolve, reject) => { + zlib.inflateRaw(data, (error, result) => { + if (error) { + reject(error); + } else { + // this.fileSources.set(index, result); + resolve(result); + } + }); + }); + } else { + const decompressedData = zlib.inflateRawSync(data); + // this.fileSources.set(index, decompressedData); + return decompressedData; + } + } + + async fchmodPromise(fd: number, mask: number): Promise { + return this.chmodPromise(this.fdToPath(fd, `fchmod`), mask); + } + + fchmodSync(fd: number, mask: number): void { + return this.chmodSync(this.fdToPath(fd, `fchmodSync`), mask); + } + + async chmodPromise(p: PortablePath, mask: number) { + return this.chmodSync(p, mask); + } + + chmodSync(p: PortablePath, mask: number) { + throw errors.EROFS(`chmod '${p}'`); + } + + async fchownPromise(fd: number, uid: number, gid: number): Promise { + return this.chownPromise(this.fdToPath(fd, `fchown`), uid, gid); + } + + fchownSync(fd: number, uid: number, gid: number): void { + return this.chownSync(this.fdToPath(fd, `fchownSync`), uid, gid); + } + + async chownPromise(p: PortablePath, uid: number, gid: number) { + return this.chownSync(p, uid, gid); + } + + chownSync(p: PortablePath, uid: number, gid: number) { + throw new Error(`Unimplemented`); + } + + async renamePromise(oldP: PortablePath, newP: PortablePath) { + return this.renameSync(oldP, newP); + } + + renameSync(oldP: PortablePath, newP: PortablePath): never { + throw new Error(`Unimplemented`); + } + + async copyFilePromise(sourceP: PortablePath, destP: PortablePath, flags?: number) { + throw errors.EROFS(`copyfile '${sourceP} -> '${destP}'`); + + } + + copyFileSync(sourceP: PortablePath, destP: PortablePath, flags: number = 0) { + throw errors.EROFS(`copyfile '${sourceP} -> '${destP}'`); + + } + + + async appendFilePromise(p: FSPath, content: string | Uint8Array, opts?: WriteFileOptions) { + throw errors.EROFS(`open '${p}'`); + } + + appendFileSync(p: FSPath, content: string | Uint8Array, opts: WriteFileOptions = {}) { + throw errors.EROFS(`open '${p}'`); + } + + private fdToPath(fd: number, reason: string) { + const path = this.fds.get(fd)?.p; + if (typeof path === `undefined`) + throw errors.EBADF(reason); + + return path; + } + + async writeFilePromise(p: FSPath, content: string | NodeJS.ArrayBufferView, opts?: WriteFileOptions) { + throw errors.EROFS(`open '${p}'`); + } + + writeFileSync(p: FSPath, content: string | NodeJS.ArrayBufferView, opts?: WriteFileOptions) { + throw errors.EROFS(`open '${p}'`); + } + + + async unlinkPromise(p: PortablePath) { + return this.unlinkSync(p); + } + + unlinkSync(p: PortablePath) { + throw errors.EROFS(`unlink '${p}'`); + } + + async utimesPromise(p: PortablePath, atime: Date | string | number, mtime: Date | string | number) { + return this.utimesSync(p, atime, mtime); + } + + utimesSync(p: PortablePath, atime: Date | string | number, mtime: Date | string | number) { + throw errors.EROFS(`utimes '${p}'`); + } + + async lutimesPromise(p: PortablePath, atime: Date | string | number, mtime: Date | string | number) { + return this.lutimesSync(p, atime, mtime); + } + + lutimesSync(p: PortablePath, atime: Date | string | number, mtime: Date | string | number) { + throw errors.EROFS(`lutimes '${p}'`); + } + + + async mkdirPromise(p: PortablePath, opts?: MkdirOptions) { + return this.mkdirSync(p, opts); + } + + mkdirSync(p: PortablePath, { mode = 0o755, recursive = false }: MkdirOptions = {}) { + throw errors.EROFS(`mkdir '${p}'`); + return undefined; + } + + async rmdirPromise(p: PortablePath, opts?: RmdirOptions) { + return this.rmdirSync(p, opts); + } + + rmdirSync(p: PortablePath, { recursive = false }: RmdirOptions = {}) { + throw errors.EROFS(`rmdir '${p}'`); + } + async rmPromise(p: PortablePath, opts?: RmOptions) { + return this.rmSync(p, opts); + } + + rmSync(p: PortablePath, { recursive = false }: RmOptions = {}) { + throw errors.EROFS(`rm '${p}'`); + + } + + + async linkPromise(existingP: PortablePath, newP: PortablePath) { + return this.linkSync(existingP, newP); + } + + linkSync(existingP: PortablePath, newP: PortablePath) { + throw errors.EROFS(`link '${existingP}' -> '${newP}'`); + } + + async symlinkPromise(target: PortablePath, p: PortablePath) { + return this.symlinkSync(target, p); + } + + symlinkSync(target: PortablePath, p: PortablePath) { + throw errors.EROFS(`symlink '${target}' -> '${p}'`); + } + + getAllFiles() { + return Array.from(this.entries.keys()); + } + + readFilePromise(p: FSPath, encoding?: null): Promise; + readFilePromise(p: FSPath, encoding: BufferEncoding): Promise; + readFilePromise(p: FSPath, encoding?: BufferEncoding | null): Promise; + async readFilePromise(p: FSPath, encoding?: BufferEncoding | null) { + // This is messed up regarding the TS signatures + if (typeof encoding === `object`) + // @ts-expect-error + encoding = encoding ? encoding.encoding : undefined; + + const data = await this.readFileBuffer(p, { asyncDecompress: true }); + return encoding ? data.toString(encoding) : data; + } + + readFileSync(p: FSPath, encoding?: null): Buffer; + readFileSync(p: FSPath, encoding: BufferEncoding): string; + readFileSync(p: FSPath, encoding?: BufferEncoding | null): Buffer | string; + readFileSync(p: FSPath, encoding?: BufferEncoding | null) { + // This is messed up regarding the TS signatures + if (typeof encoding === `object`) + // @ts-expect-error + encoding = encoding ? encoding.encoding : undefined; + + const data = this.readFileBuffer(p); + return encoding ? data.toString(encoding) : data; + } + + private readFileBuffer(p: FSPath): Buffer; + private readFileBuffer(p: FSPath, opts: { asyncDecompress: false }): Buffer; + private readFileBuffer(p: FSPath, opts: { asyncDecompress: true }): Promise; + private readFileBuffer(p: FSPath, opts: { asyncDecompress: boolean }): Promise | Buffer; + private readFileBuffer(p: FSPath, opts: { asyncDecompress: boolean } = { asyncDecompress: false }): Buffer | Promise { + if (typeof p === `number`) + p = this.fdToPath(p, `read`); + + const resolvedP = this.resolveFilename(`open '${p}'`, p); + if (!this.entries.has(resolvedP) && !this.listings.has(resolvedP)) + throw errors.ENOENT(`open '${p}'`); + + // Ensures that the last component is a directory, if the user said so (even if it is we'll throw right after with EISDIR anyway) + if (p[p.length - 1] === `/` && !this.listings.has(resolvedP)) + throw errors.ENOTDIR(`open '${p}'`); + + if (this.listings.has(resolvedP)) + throw errors.EISDIR(`read`); + + const entry = this.entries.get(resolvedP); + if (entry === undefined) + throw new Error(`Unreachable`); + + return this.getFileSource(entry, opts); + } + + async readdirPromise(p: PortablePath, opts?: null): Promise>; + async readdirPromise(p: PortablePath, opts: { recursive?: false, withFileTypes: true }): Promise>; + async readdirPromise(p: PortablePath, opts: { recursive?: false, withFileTypes?: false }): Promise>; + async readdirPromise(p: PortablePath, opts: { recursive?: false, withFileTypes: boolean }): Promise>; + async readdirPromise(p: PortablePath, opts: { recursive: true, withFileTypes: true }): Promise>>; + async readdirPromise(p: PortablePath, opts: { recursive: true, withFileTypes?: false }): Promise>; + async readdirPromise(p: PortablePath, opts: { recursive: true, withFileTypes: boolean }): Promise | PortablePath>>; + async readdirPromise(p: PortablePath, opts: { recursive: boolean, withFileTypes: true }): Promise | DirentNoPath>>; + async readdirPromise(p: PortablePath, opts: { recursive: boolean, withFileTypes?: false }): Promise>; + async readdirPromise(p: PortablePath, opts: { recursive: boolean, withFileTypes: boolean }): Promise | DirentNoPath | PortablePath>>; + async readdirPromise(p: PortablePath, opts?: ReaddirOptions | null): Promise | DirentNoPath | PortablePath>> { + return this.readdirSync(p, opts as any); + } + + readdirSync(p: PortablePath, opts?: null): Array; + readdirSync(p: PortablePath, opts: { recursive?: false, withFileTypes: true }): Array; + readdirSync(p: PortablePath, opts: { recursive?: false, withFileTypes?: false }): Array; + readdirSync(p: PortablePath, opts: { recursive?: false, withFileTypes: boolean }): Array; + readdirSync(p: PortablePath, opts: { recursive: true, withFileTypes: true }): Array>; + readdirSync(p: PortablePath, opts: { recursive: true, withFileTypes?: false }): Array; + readdirSync(p: PortablePath, opts: { recursive: true, withFileTypes: boolean }): Array | PortablePath>; + readdirSync(p: PortablePath, opts: { recursive: boolean, withFileTypes: true }): Array | DirentNoPath>; + readdirSync(p: PortablePath, opts: { recursive: boolean, withFileTypes?: false }): Array; + readdirSync(p: PortablePath, opts: { recursive: boolean, withFileTypes: boolean }): Array | DirentNoPath | PortablePath>; + readdirSync(p: PortablePath, opts?: ReaddirOptions | null): Array | DirentNoPath | PortablePath> { + const resolvedP = this.resolveFilename(`scandir '${p}'`, p); + if (!this.entries.has(resolvedP) && !this.listings.has(resolvedP)) + throw errors.ENOENT(`scandir '${p}'`); + + const directoryListing = this.listings.get(resolvedP); + if (!directoryListing) + throw errors.ENOTDIR(`scandir '${p}'`); + + if (opts?.recursive) { + if (opts?.withFileTypes) { + const entries = Array.from(directoryListing, name => { + return Object.assign(this.statImpl(`lstat`, ppath.join(p, name)), { + name, + path: PortablePath.dot, + }); + }); + + for (const entry of entries) { + if (!entry.isDirectory()) + continue; + + const subPath = ppath.join(entry.path, entry.name); + const subListing = this.listings.get(ppath.join(resolvedP, subPath))!; + + for (const child of subListing) { + entries.push(Object.assign(this.statImpl(`lstat`, ppath.join(p, subPath, child)), { + name: child, + path: subPath, + })); + } + } + + return entries; + } else { + const entries: Array = [...directoryListing]; + + for (const subPath of entries) { + const subListing = this.listings.get(ppath.join(resolvedP, subPath)); + if (typeof subListing === `undefined`) + continue; + + for (const child of subListing) { + entries.push(ppath.join(subPath, child)); + } + } + + return entries; + } + } else if (opts?.withFileTypes) { + return Array.from(directoryListing, name => { + return Object.assign(this.statImpl(`lstat`, ppath.join(p, name)), { + name, + path: undefined, + }); + }); + } else { + return [...directoryListing]; + } + } + + async readlinkPromise(p: PortablePath) { + const entry = this.prepareReadlink(p); + return (await this.getFileSource(entry, { asyncDecompress: true })).toString() as PortablePath; + } + + readlinkSync(p: PortablePath): PortablePath { + const entry = this.prepareReadlink(p); + return this.getFileSource(entry).toString() as PortablePath; + } + + private prepareReadlink(p: PortablePath) { + const resolvedP = this.resolveFilename(`readlink '${p}'`, p, false); + if (!this.entries.has(resolvedP) && !this.listings.has(resolvedP)) + throw errors.ENOENT(`readlink '${p}'`); + + // Ensure that the last component is a directory (if it is we'll throw right after with EISDIR anyway) + if (p[p.length - 1] === `/` && !this.listings.has(resolvedP)) + throw errors.ENOTDIR(`open '${p}'`); + + if (this.listings.has(resolvedP)) + throw errors.EINVAL(`readlink '${p}'`); + + const entry = this.entries.get(resolvedP); + if (entry === undefined) + throw new Error(`Unreachable`); + + if (!this.isSymbolicLink(entry)) + throw errors.EINVAL(`readlink '${p}'`); + + return entry; + } + + async truncatePromise(p: PortablePath, len: number = 0) { + const resolvedP = this.resolveFilename(`open '${p}'`, p); + + const index = this.entries.get(resolvedP); + if (typeof index === `undefined`) + throw errors.EINVAL(`open '${p}'`); + + const source = await this.getFileSource(index, { asyncDecompress: true }); + + const truncated = Buffer.alloc(len, 0x00); + source.copy(truncated); + + return await this.writeFilePromise(p, truncated); + } + + truncateSync(p: PortablePath, len: number = 0) { + const resolvedP = this.resolveFilename(`open '${p}'`, p); + + const index = this.entries.get(resolvedP); + if (typeof index === `undefined`) + throw errors.EINVAL(`open '${p}'`); + + const source = this.getFileSource(index); + + const truncated = Buffer.alloc(len, 0x00); + source.copy(truncated); + + return this.writeFileSync(p, truncated); + } + + async ftruncatePromise(fd: number, len?: number): Promise { + return this.truncatePromise(this.fdToPath(fd, `ftruncate`), len); + } + + ftruncateSync(fd: number, len?: number): void { + return this.truncateSync(this.fdToPath(fd, `ftruncateSync`), len); + } + + watch(p: PortablePath, cb?: WatchCallback): Watcher; + watch(p: PortablePath, opts: WatchOptions, cb?: WatchCallback): Watcher; + watch(p: PortablePath, a?: WatchOptions | WatchCallback, b?: WatchCallback) { + let persistent: boolean; + + switch (typeof a) { + case `function`: + case `string`: + case `undefined`: { + persistent = true; + } break; + + default: { + ({ persistent = true } = a); + } break; + } + + if (!persistent) + return { on: () => { }, close: () => { } }; + + const interval = setInterval(() => { }, 24 * 60 * 60 * 1000); + return { + on: () => { }, close: () => { + clearInterval(interval); + } + }; + } + + watchFile(p: PortablePath, cb: WatchFileCallback): StatWatcher; + watchFile(p: PortablePath, opts: WatchFileOptions, cb: WatchFileCallback): StatWatcher; + watchFile(p: PortablePath, a: WatchFileOptions | WatchFileCallback, b?: WatchFileCallback) { + const resolvedP = ppath.resolve(PortablePath.root, p); + + return watchFile(this, resolvedP, a, b); + } + + unwatchFile(p: PortablePath, cb?: WatchFileCallback): void { + const resolvedP = ppath.resolve(PortablePath.root, p); + + return unwatchFile(this, resolvedP, cb); + } +} diff --git a/packages/yarnpkg-minizip/sources/MiniZipOpenFS.ts b/packages/yarnpkg-minizip/sources/MiniZipOpenFS.ts new file mode 100644 index 000000000000..d3458571d6b1 --- /dev/null +++ b/packages/yarnpkg-minizip/sources/MiniZipOpenFS.ts @@ -0,0 +1,111 @@ +import {FakeFS} from '@yarnpkg/fslib'; +import {GetMountPointFn, MountFS, MountFSOptions} from '@yarnpkg/fslib'; +import {PortablePath, ppath} from '@yarnpkg/fslib'; + +import {MiniZipFS} from './MiniZipFS'; +/** + * Extracts the archive part (ending in the first instance of `extension`) from a path. + * + * The indexOf-based implementation is ~3.7x faster than a RegExp-based implementation. + */ +export function getArchivePart(path: string, extension: string) { + let idx = path.indexOf(extension); + if (idx <= 0) + return null; + + let nextCharIdx = idx; + while (idx >= 0) { + nextCharIdx = idx + extension.length; + if (path[nextCharIdx] === ppath.sep) + break; + + // Disallow files named ".zip" + if (path[idx - 1] === ppath.sep) + return null; + + idx = path.indexOf(extension, nextCharIdx); + } + + // The path either has to end in ".zip" or contain an archive subpath (".zip/...") + if (path.length > nextCharIdx && path[nextCharIdx] !== ppath.sep) + return null; + + return path.slice(0, nextCharIdx) as PortablePath; +} + +export type ZipOpenFSOptions = Omit, + | `factoryPromise` + | `factorySync` + | `getMountPoint` +> & { + + readOnlyArchives?: true; + + /** + * Which file extensions will be interpreted as zip files. Useful for supporting other formats + * packaged as zips, such as .docx. + * + * If not provided, defaults to only accepting `.zip`. + */ + fileExtensions?: Array | null; +}; + +export class MiniZipOpenFS extends MountFS { + static async openPromise(fn: (zipOpenFs: MiniZipOpenFS) => Promise, opts?: ZipOpenFSOptions): Promise { + const zipOpenFs = new MiniZipOpenFS(opts); + + try { + return await fn(zipOpenFs); + } finally { + zipOpenFs.saveAndClose(); + } + } + + constructor(opts: ZipOpenFSOptions = {}) { + if (opts.readOnlyArchives !== true) { + throw new Error(`The MiniZipOpenFS can only be used with read-only archives`); + } + const fileExtensions = opts.fileExtensions; + const readOnlyArchives = opts.readOnlyArchives; + + const getMountPoint: GetMountPointFn = typeof fileExtensions === `undefined` + ? path => getArchivePart(path, `.zip`) + : path => { + for (const extension of fileExtensions!) { + const result = getArchivePart(path, extension); + if (result) { + return result; + } + } + + return null; + }; + + const factorySync = (baseFs: FakeFS, p: PortablePath) => { + return new MiniZipFS(p, { + baseFs, + }); + }; + + const factoryPromise = async (baseFs: FakeFS, p: PortablePath) => { + const zipOptions = { + baseFs, + readOnly: readOnlyArchives, + stats: await baseFs.statPromise(p), + }; + + return () => { + return new MiniZipFS(p, zipOptions); + }; + }; + + super({ + ...opts, + + factorySync, + factoryPromise, + + getMountPoint, + }); + } +} diff --git a/packages/yarnpkg-minizip/sources/index.ts b/packages/yarnpkg-minizip/sources/index.ts new file mode 100644 index 000000000000..dc792f2fa696 --- /dev/null +++ b/packages/yarnpkg-minizip/sources/index.ts @@ -0,0 +1,2 @@ +export { MiniZipOpenFS } from './MiniZipOpenFS' +export { MiniZipFS } from './MiniZipFS' \ No newline at end of file diff --git a/packages/yarnpkg-minizip/tests/ZipFS.test.ts b/packages/yarnpkg-minizip/tests/ZipFS.test.ts new file mode 100644 index 000000000000..b73276f88184 --- /dev/null +++ b/packages/yarnpkg-minizip/tests/ZipFS.test.ts @@ -0,0 +1,1028 @@ +import {Filename, PortablePath, constants, ppath, statUtils, xfs} from '@yarnpkg/fslib'; +import {makeEmptyArchive, ZipFS} from '@yarnpkg/libzip'; +import {S_IFREG} from 'constants'; +import fs from 'fs'; + +const isNotWin32 = process.platform !== `win32`; + +const ifNotWin32It = isNotWin32 + ? it + : it.skip; + +afterEach(() => { + jest.useRealTimers(); +}); + +describe(`ZipFS`, () => { + it(`should handle symlink correctly`, () => { + const expectSameStats = (a: fs.Stats, b: fs.Stats) => { + expect(a.ino).toEqual(b.ino); + expect(a.size).toEqual(b.size); + expect(a.mode).toEqual(b.mode); + expect(a.atimeMs).toEqual(b.atimeMs); + expect(a.mtimeMs).toEqual(b.mtimeMs); + expect(a.ctimeMs).toEqual(b.ctimeMs); + expect(a.birthtimeMs).toEqual(b.birthtimeMs); + expect(a.isFile()).toEqual(a.isFile()); + expect(a.isDirectory()).toEqual(a.isDirectory()); + expect(a.isSymbolicLink()).toEqual(a.isSymbolicLink()); + }; + + const asserts = (zipFs: ZipFS) => { + const dir = zipFs.statSync(`/dir` as PortablePath); + expect(dir.isFile()).toBeFalsy(); + expect(dir.isDirectory()).toBeTruthy(); + expect(dir.isSymbolicLink()).toBeFalsy(); + + const file = zipFs.statSync(`/dir/file` as PortablePath); + expect(file.isFile()).toBeTruthy(); + expect(file.isDirectory()).toBeFalsy(); + expect(file.isSymbolicLink()).toBeFalsy(); + + expectSameStats(zipFs.lstatSync(`/dir/file` as PortablePath), file); + expectSameStats(zipFs.lstatSync(`/dir` as PortablePath), dir); + + expectSameStats(zipFs.statSync(`/linkToFileA` as PortablePath), file); + expectSameStats(zipFs.statSync(`/linkToFileB` as PortablePath), file); + expectSameStats(zipFs.statSync(`/linkToDirA/file` as PortablePath), file); + expectSameStats(zipFs.statSync(`/linkToDirB/file` as PortablePath), file); + expectSameStats(zipFs.statSync(`/linkToCwd/linkToCwd/linkToCwd/linkToCwd/dir/file` as PortablePath), file); + + expectSameStats(zipFs.statSync(`/linkToDirA` as PortablePath), dir); + expectSameStats(zipFs.statSync(`/linkToDirB` as PortablePath), dir); + expectSameStats(zipFs.statSync(`/linkToCwd/linkToCwd/linkToCwd/linkToCwd/linkToDirA` as PortablePath), dir); + + expectSameStats(zipFs.lstatSync(`/linkToDirA/file` as PortablePath), file); + expectSameStats(zipFs.lstatSync(`/linkToDirB/file` as PortablePath), file); + expectSameStats(zipFs.lstatSync(`/linkToCwd/linkToCwd/linkToCwd/linkToCwd/dir/file` as PortablePath), file); + + const linkToDirA = zipFs.lstatSync(`/linkToDirA` as PortablePath); + expect(linkToDirA.isFile()).toBeFalsy(); + expect(linkToDirA.isDirectory()).toBeFalsy(); + expect(linkToDirA.isSymbolicLink()).toBeTruthy(); + + const linkToDirB = zipFs.lstatSync(`/linkToDirB` as PortablePath); + expect(linkToDirB.isFile()).toBeFalsy(); + expect(linkToDirB.isDirectory()).toBeFalsy(); + expect(linkToDirB.isSymbolicLink()).toBeTruthy(); + + for (const path of [ + `/linkToFileA`, + `/linkToFileB`, + `/linkToDirA/file`, + `/linkToDirB/file`, + `/dir/file`, + `/linkToCwd/linkToCwd/linkToCwd/linkToCwd/dir/file`, + ]) + expect(zipFs.readFileSync(path as PortablePath, `utf8`)).toEqual(`file content`); + + + for (const path of [ + `/linkToDirA`, + `/linkToDirB`, + `/linkToCwd/linkToCwd/linkToCwd/linkToCwd/dir`, + `/linkToCwd/linkToCwd/linkToCwd/linkToCwd/linkToDirA`, + `/linkToCwd/linkToCwd/linkToCwd/linkToCwd/linkToDirB`, + ]) { + expect(zipFs.readdirSync(path as PortablePath)).toContain(`file`); + } + }; + + const tmpfile = ppath.resolve(xfs.mktempSync(), `test.zip`); + const zipFs = new ZipFS(tmpfile, {create: true}); + + zipFs.mkdirSync(`/dir` as PortablePath); + zipFs.writeFileSync(`/dir/file` as PortablePath, `file content`); + + zipFs.symlinkSync(`dir/file` as PortablePath, `linkToFileA` as PortablePath); + zipFs.symlinkSync(`./dir/file` as PortablePath, `linkToFileB` as PortablePath); + zipFs.symlinkSync(`dir` as PortablePath, `linkToDirA` as PortablePath); + zipFs.symlinkSync(`./dir` as PortablePath, `linkToDirB` as PortablePath); + zipFs.symlinkSync(`.` as PortablePath, `linkToCwd` as PortablePath); + + // asserts(zipFs); + zipFs.saveAndClose(); + + const zipFs2 = new ZipFS(tmpfile); + asserts(zipFs2); + zipFs2.discardAndClose(); + }); + + it(`should readSync file contents`, async () => { + const readFileContents = function (zipFs: ZipFS, p: PortablePath, position: number | null) { + const fd = zipFs.openSync(p, `r`); + const buffer = Buffer.alloc(8192); + try { + let size = 0; + let read = 0; + while ((read = zipFs.readSync(fd, buffer, 0, buffer.length, position)) !== 0) + size += read; + + return buffer.toString(`utf-8`, 0, size); + } finally { + zipFs.closeSync(fd); + } + }; + const readSyncAsserts = (zipFs: ZipFS) => { + const p = `/dir/file` as PortablePath; + expect(readFileContents(zipFs, p, -1)).toEqual(`file content`); + expect(readFileContents(zipFs, p, null)).toEqual(`file content`); + }; + + const tmpfile = ppath.resolve(xfs.mktempSync(), `test2.zip`); + const zipFs = new ZipFS(tmpfile, {create: true}); + await zipFs.mkdirPromise(`/dir` as PortablePath); + zipFs.writeFileSync(`/dir/file` as PortablePath, `file content`); + zipFs.saveAndClose(); + + const zipFs2 = new ZipFS(tmpfile); + readSyncAsserts(zipFs2); + zipFs2.discardAndClose(); + }); + + it(`defaults the readSync read length to the buffer size`, async () => { + const p = `/dir/file` as PortablePath; + const zipFs = new ZipFS(); + await zipFs.mkdirPromise(`/dir` as PortablePath); + zipFs.writeFileSync(p, `file content`); + + const buffer = Buffer.alloc(8192); + const fd = zipFs.openSync(p, `r`); + try { + zipFs.readSync(fd, buffer); + expect(buffer.slice(0, buffer.indexOf(`\0`)).toString()).toEqual(`file content`); + } finally { + zipFs.closeSync(fd); + } + zipFs.discardAndClose(); + }); + + it(`can create a zip file in memory`, () => { + const zipFs = new ZipFS(); + + zipFs.writeFileSync(`/foo.txt` as PortablePath, `Test`); + + const zipContent = zipFs.getBufferAndClose(); + + const zipFs2 = new ZipFS(zipContent); + expect(zipFs2.readFileSync(`/foo.txt` as PortablePath, `utf8`)).toEqual(`Test`); + }); + + it(`can handle nested symlinks`, () => { + const zipFs = new ZipFS(); + zipFs.writeFileSync(`/foo.txt` as PortablePath, `Test`); + + zipFs.symlinkSync(`/foo.txt` as PortablePath, `/linkA` as PortablePath); + zipFs.symlinkSync(`/linkA` as PortablePath, `/linkB` as PortablePath); + + const zipFs2 = new ZipFS(zipFs.getBufferAndClose()); + + expect(zipFs2.readFileSync(`/linkA` as PortablePath, `utf8`)).toEqual(`Test`); + expect(zipFs2.readFileSync(`/linkB` as PortablePath, `utf8`)).toEqual(`Test`); + + zipFs2.discardAndClose(); + }); + + it(`returns the same content for sync and async reads`, async () => { + const zipFs = new ZipFS(); + zipFs.writeFileSync(`/foo.txt` as PortablePath, `Test`); + + const zipFs2 = new ZipFS(zipFs.getBufferAndClose()); + + expect(await zipFs2.readFilePromise(`/foo.txt` as PortablePath, `utf8`)).toEqual(`Test`); + expect(zipFs2.readFileSync(`/foo.txt` as PortablePath, `utf8`)).toEqual(`Test`); + }); + + it(`should support unlinking files`, () => { + const zipFs = new ZipFS(); + + const dir = `/foo` as PortablePath; + zipFs.mkdirSync(dir); + + const file = `/foo/bar.txt` as PortablePath; + zipFs.writeFileSync(file, `Test`); + + expect(zipFs.existsSync(dir)).toBeTruthy(); + expect(zipFs.existsSync(file)).toBeTruthy(); + + zipFs.unlinkSync(file); + + expect(zipFs.existsSync(dir)).toBeTruthy(); + expect(zipFs.existsSync(file)).toBeFalsy(); + + zipFs.discardAndClose(); + }); + + it(`should support removing empty directories`, () => { + const zipFs = new ZipFS(); + + const dir = `/foo` as PortablePath; + const subdir = `/foo/bar` as PortablePath; + zipFs.mkdirpSync(subdir); + + expect(zipFs.existsSync(dir)).toBeTruthy(); + expect(zipFs.existsSync(subdir)).toBeTruthy(); + + zipFs.rmdirSync(subdir); + + expect(zipFs.existsSync(dir)).toBeTruthy(); + expect(zipFs.existsSync(subdir)).toBeFalsy(); + + zipFs.discardAndClose(); + }); + + it(`should not support removing non-empty directories`, () => { + const zipFs = new ZipFS(); + + const dir = `/foo` as PortablePath; + zipFs.mkdirSync(dir); + + const file = `/foo/bar.txt` as PortablePath; + zipFs.writeFileSync(file, `Test`); + + expect(() => zipFs.rmdirSync(dir)).toThrowError(`ENOTEMPTY`); + + zipFs.discardAndClose(); + }); + + it(`should support removing non-empty directories via zipFs.removeSync`, () => { + const zipFs = new ZipFS(); + + const dir = `/foo` as PortablePath; + const subdir = `/foo/bar` as PortablePath; + zipFs.mkdirpSync(subdir); + + const file = `/foo/bar/baz.txt` as PortablePath; + zipFs.writeFileSync(file, `Test`); + + expect(zipFs.existsSync(dir)).toBeTruthy(); + expect(zipFs.existsSync(subdir)).toBeTruthy(); + expect(zipFs.existsSync(file)).toBeTruthy(); + + zipFs.removeSync(subdir); + + expect(zipFs.existsSync(dir)).toBeTruthy(); + expect(zipFs.existsSync(subdir)).toBeFalsy(); + expect(zipFs.existsSync(file)).toBeFalsy(); + }); + + it(`should support read after write`, () => { + const zipFs = new ZipFS(); + + const file = `/foo.txt` as PortablePath; + zipFs.writeFileSync(file, `Test`); + + expect(zipFs.readFileSync(file, `utf8`)).toStrictEqual(`Test`); + + zipFs.discardAndClose(); + }); + + it(`should support write after read`, () => { + const tmpdir = xfs.mktempSync(); + const archive = `${tmpdir}/archive.zip` as PortablePath; + + const zipFs = new ZipFS(archive, {create: true}); + + const file = `/foo.txt` as PortablePath; + zipFs.writeFileSync(file, `Hello World`); + + zipFs.saveAndClose(); + + const zipFs2 = new ZipFS(archive); + + expect(zipFs2.readFileSync(file, `utf8`)).toStrictEqual(`Hello World`); + expect(() => zipFs2.writeFileSync(file, `Goodbye World`)).not.toThrow(); + + zipFs2.discardAndClose(); + }); + + it(`should support write after write`, () => { + const zipFs = new ZipFS(); + + const file = `/foo.txt` as PortablePath; + + zipFs.writeFileSync(file, `Hello World`); + expect(() => zipFs.writeFileSync(file, `Goodbye World`)).not.toThrow(); + + zipFs.discardAndClose(); + }); + + it(`should support read after read`, () => { + const tmpdir = xfs.mktempSync(); + const archive = `${tmpdir}/archive.zip` as PortablePath; + + const zipFs = new ZipFS(archive, {create: true}); + + const file = `/foo.txt` as PortablePath; + zipFs.writeFileSync(file, `Hello World`); + + zipFs.saveAndClose(); + + const zipFs2 = new ZipFS(archive); + + expect(zipFs2.readFileSync(file, `utf8`)).toStrictEqual(`Hello World`); + expect(zipFs2.readFileSync(file, `utf8`)).toStrictEqual(`Hello World`); + + zipFs2.discardAndClose(); + }); + + it(`should support truncate`, () => { + const zipFs = new ZipFS(); + + const file = `/foo.txt` as PortablePath; + + zipFs.writeFileSync(file, `1234567890`); + + zipFs.truncateSync(file, 5); + expect(zipFs.readFileSync(file, `utf8`)).toStrictEqual(`12345`); + + zipFs.truncateSync(file, 10); + expect(zipFs.readFileSync(file, `utf8`)).toStrictEqual(`12345${`\u0000`.repeat(5)}`); + + zipFs.truncateSync(file); + expect(zipFs.readFileSync(file, `utf8`)).toStrictEqual(``); + + zipFs.discardAndClose(); + }); + + it(`should support ftruncate`, async () => { + const zipFs = new ZipFS(); + + const fd = zipFs.openSync(`/foo.txt` as PortablePath, `r+`); + + zipFs.writeFileSync(fd, `1234567890`); + + zipFs.ftruncateSync(fd, 5); + expect(zipFs.readFileSync(fd, `utf8`)).toStrictEqual(`12345`); + + await zipFs.ftruncatePromise(fd, 4); + expect(zipFs.readFileSync(fd, `utf8`)).toStrictEqual(`1234`); + + zipFs.closeSync(fd); + zipFs.discardAndClose(); + }); + + it(`should support watchFile and unwatchFile`, () => { + const zipFs = new ZipFS(); + + const file = `/foo.txt` as PortablePath; + + const emptyStats = statUtils.makeEmptyStats(); + + const changeListener = jest.fn(); + const stopListener = jest.fn(); + + jest.useFakeTimers(); + + const statWatcher = zipFs.watchFile(file, {interval: 1000}, changeListener); + statWatcher.on(`stop`, stopListener); + + // The listener should be initially called with empty stats if the path doesn't exist, + // but only after 3 milliseconds, so that other listeners can be registered in that timespan + // (That's what Node does) + + expect(changeListener).not.toHaveBeenCalled(); + + jest.advanceTimersByTime(3); + + expect(changeListener).toHaveBeenCalledTimes(1); + expect(changeListener).toHaveBeenCalledWith(emptyStats, emptyStats); + + // The watcher should pick up changes in content + + zipFs.writeFileSync(file, `Hello World`); + const first = zipFs.statSync(file); + + jest.advanceTimersByTime(1000); + + expect(changeListener).toHaveBeenCalledTimes(2); + expect(changeListener).toHaveBeenCalledWith(first, emptyStats); + + // The watcher should only pick up the last changes in an interval + + zipFs.writeFileSync(file, `This shouldn't be picked up`); + + zipFs.writeFileSync(file, `Goodbye World`); + const second = zipFs.statSync(file); + + jest.advanceTimersByTime(1000); + + expect(changeListener).toHaveBeenCalledTimes(3); + expect(changeListener).toHaveBeenCalledWith(second, first); + + // The watcher should pick up deletions + + zipFs.unlinkSync(file); + + jest.advanceTimersByTime(1000); + + expect(changeListener).toHaveBeenCalledTimes(4); + expect(changeListener).toHaveBeenCalledWith(emptyStats, second); + + // unwatchFile should work + + expect(stopListener).not.toHaveBeenCalled(); + + zipFs.unwatchFile(file, changeListener); + + // The stop event should be emitted when there are no remaining change listeners + expect(stopListener).toHaveBeenCalledTimes(1); + + // The listener shouldn't be called after the file is unwatched + + zipFs.writeFileSync(file, `Test`); + + jest.advanceTimersByTime(1000); + + expect(changeListener).toHaveBeenCalledTimes(4); + + zipFs.discardAndClose(); + + // The watcher shouldn't keep the process running after the file is unwatched + }); + + it(`should accept invalid paths on watchFile (ENOTDIR)`, async () => { + const zipFs = new ZipFS(); + + const file = `/foo.txt/package.json` as PortablePath; + + // Should cause a ENOTDIR error to trigger, but watchFile doesn't care + zipFs.writeFileSync(ppath.dirname(file), ``); + + const emptyStats = statUtils.makeEmptyStats(); + + const changeListener = jest.fn(); + const stopListener = jest.fn(); + + jest.useFakeTimers(); + + const statWatcher = zipFs.watchFile(file, {interval: 1000}, changeListener); + statWatcher.on(`stop`, stopListener); + + expect(changeListener).not.toHaveBeenCalled(); + + jest.advanceTimersByTime(3); + + expect(changeListener).toHaveBeenCalledTimes(1); + expect(changeListener).toHaveBeenCalledWith(emptyStats, emptyStats); + + zipFs.discardAndClose(); + }); + + it(`closes the fd created in createReadStream when the stream is closed early`, async () => { + const zipFs = new ZipFS(); + zipFs.writeFileSync(`/foo.txt` as Filename, `foo`.repeat(10000)); + + expect(zipFs.hasOpenFileHandles()).toBe(false); + const stream = zipFs.createReadStream(`/foo.txt` as Filename); + + expect(zipFs.hasOpenFileHandles()).toBe(true); + + await new Promise((resolve, reject) => { + stream.on(`data`, () => { + reject(new Error(`Should not be called`)); + }); + stream.on(`close`, () => { + resolve(); + }); + stream.on(`error`, error => { + reject(error); + }); + + stream.close(); + }); + + expect(zipFs.hasOpenFileHandles()).toBe(false); + + zipFs.discardAndClose(); + }); + + it(`should close the createWriteStream when destroyed`, async () => { + const zipFs = new ZipFS(); + + const writeStream = zipFs.createWriteStream(`/foo.txt` as Filename); + + await new Promise((resolve, reject) => { + writeStream.write(`foo`, err => { + if (err) { + reject(err); + } else { + writeStream.destroy(); + resolve(); + } + }); + }); + + expect(zipFs.hasOpenFileHandles()).toBe(false); + + expect(zipFs.readFileSync(`/foo.txt` as Filename, `utf8`)).toBe(`foo`); + + zipFs.discardAndClose(); + }); + + it(`should stop the watcher on closing the archive`, async () => { + jest.useFakeTimers(); + const zipFs = new ZipFS(); + + zipFs.writeFileSync(`/foo.txt` as PortablePath, `foo`); + + zipFs.watchFile(`/foo.txt` as PortablePath, (current, previous) => {}); + + zipFs.discardAndClose(); + + // If the watcher wasn't stopped this will trigger `EBUSY: archive closed` + jest.advanceTimersByTime(100); + }); + + it(`should support opendir`, async () => { + const zipFs = new ZipFS(); + + const folder = `/foo` as PortablePath; + zipFs.mkdirSync(folder); + + const firstFile = `/foo/1.txt` as PortablePath; + const secondFile = `/foo/2.txt` as PortablePath; + const thirdFile = `/foo/3.txt` as PortablePath; + + zipFs.writeFileSync(firstFile, ``); + zipFs.writeFileSync(secondFile, ``); + zipFs.writeFileSync(thirdFile, ``); + + const dir = zipFs.opendirSync(folder); + + expect(dir.path).toStrictEqual(folder); + + const iter = dir[Symbol.asyncIterator](); + + expect((await iter.next()).value.name).toStrictEqual(ppath.basename(firstFile)); + expect(dir.readSync()!.name).toStrictEqual(ppath.basename(secondFile)); + expect((await dir.read())!.name).toStrictEqual(ppath.basename(thirdFile)); + + expect((await iter.next()).value).toBeUndefined(); + + // Consuming the iterator should cause the Dir instance to close + + // FIXME: This assertion fails + // await expect(() => iter.next()).rejects.toThrow(`Directory handle was closed`); + expect(() => dir.readSync()).toThrow(`Directory handle was closed`); + // It's important that this function throws synchronously, because that's what Node does + expect(() => dir.read()).toThrow(`Directory handle was closed`); + + zipFs.discardAndClose(); + }); + + it(`closes the fd created in opendir when the Dir is closed early`, () => { + const zipFs = new ZipFS(); + zipFs.mkdirSync(`/foo` as PortablePath); + + expect(zipFs.hasOpenFileHandles()).toBe(false); + const dir = zipFs.opendirSync(`/foo` as Filename); + expect(zipFs.hasOpenFileHandles()).toBe(true); + dir.closeSync(); + expect(zipFs.hasOpenFileHandles()).toBe(false); + + zipFs.discardAndClose(); + }); + + it(`should emit the 'end' event from large reads in createReadStream`, async () => { + const zipFs = new ZipFS(); + zipFs.writeFileSync(`/foo.txt` as Filename, `foo`.repeat(10000)); + + const stream = zipFs.createReadStream(`/foo.txt` as Filename); + + let endEmitted = false; + + await new Promise((resolve, reject) => { + stream.on(`end`, () => { + endEmitted = true; + }); + + stream.on(`close`, () => { + if (!endEmitted) { + setTimeout(() => { + resolve(); + }, 1000); + } + }); + + const nullStream = fs.createWriteStream(process.platform === `win32` ? `\\\\.\\NUL` : `/dev/null`); + + const piped = stream.pipe(nullStream); + + piped.on(`finish`, () => { + resolve(); + }); + + stream.on(`error`, error => reject(error)); + piped.on(`error`, error => reject(error)); + }); + + expect(endEmitted).toBe(true); + + zipFs.discardAndClose(); + }); + + it(`should return bigint stats`, () => { + const zipFs = new ZipFS(); + zipFs.mkdirSync(`/foo` as PortablePath); + + expect( + statUtils.areStatsEqual( + zipFs.statSync(`/foo` as PortablePath, {bigint: true}), + zipFs.statSync(`/foo` as PortablePath, {bigint: true}), + ), + ).toBe(true); + + expect( + statUtils.areStatsEqual( + zipFs.statSync(`/foo` as PortablePath, {bigint: false}), + zipFs.statSync(`/foo` as PortablePath, {bigint: true}), + ), + ).toBe(false); + + zipFs.discardAndClose(); + }); + + it(`should support saving an empty zip archive`, () => { + const tmpdir = xfs.mktempSync(); + const archive = `${tmpdir}/archive.zip` as PortablePath; + + const zipFs = new ZipFS(archive, {create: true}); + zipFs.saveAndClose(); + + expect(xfs.existsSync(archive)).toStrictEqual(true); + expect(new ZipFS(archive).readdirSync(PortablePath.root)).toHaveLength(0); + }); + + it(`should support saving an empty zip archive (unlink after write)`, () => { + const tmpdir = xfs.mktempSync(); + const archive = `${tmpdir}/archive.zip` as PortablePath; + + const zipFs = new ZipFS(archive, {create: true}); + + zipFs.writeFileSync(`/foo.txt` as PortablePath, `foo`); + zipFs.unlinkSync(`/foo.txt` as PortablePath); + + zipFs.saveAndClose(); + + expect(xfs.existsSync(archive)).toStrictEqual(true); + expect(new ZipFS(archive).readdirSync(PortablePath.root)).toHaveLength(0); + }); + + it(`should support getting the buffer from an empty in-memory zip archive`, () => { + const zipFs = new ZipFS(); + const buffer = zipFs.getBufferAndClose(); + + expect(buffer).toStrictEqual(makeEmptyArchive()); + + expect(new ZipFS(buffer).readdirSync(PortablePath.root)).toHaveLength(0); + }); + + it(`should support getting the buffer from an empty in-memory zip archive (unlink after write)`, () => { + const zipFs = new ZipFS(); + + zipFs.writeFileSync(`/foo.txt` as PortablePath, `foo`); + zipFs.unlinkSync(`/foo.txt` as PortablePath); + + const buffer = zipFs.getBufferAndClose(); + + expect(buffer).toStrictEqual(makeEmptyArchive()); + + expect(new ZipFS(buffer).readdirSync(PortablePath.root)).toHaveLength(0); + }); + + ifNotWin32It(`should preserve the umask`, async () => { + const tmpdir = xfs.mktempSync(); + const archive = `${tmpdir}/archive.zip` as PortablePath; + + await xfs.writeFilePromise(archive, makeEmptyArchive(), {mode: 0o754}); + + const zipFs = new ZipFS(archive); + await zipFs.writeFilePromise(`/foo.txt` as PortablePath, `foo`); + + zipFs.saveAndClose(); + + expect((await xfs.statPromise(archive)).mode & 0o777).toStrictEqual(0o754); + }); + + ifNotWin32It(`should preserve the umask (empty archive)`, async () => { + const tmpdir = xfs.mktempSync(); + const archive = `${tmpdir}/archive.zip` as PortablePath; + + await xfs.writeFilePromise(archive, makeEmptyArchive(), {mode: 0o754}); + + const zipFs = new ZipFS(archive); + + zipFs.saveAndClose(); + + expect((await xfs.statPromise(archive)).mode & 0o777).toStrictEqual(0o754); + }); + + ifNotWin32It(`should preserve the umask if the archive is unlinked before being closed`, async () => { + const tmpdir = xfs.mktempSync(); + const archive = `${tmpdir}/archive.zip` as PortablePath; + + await xfs.writeFilePromise(archive, makeEmptyArchive(), {mode: 0o754}); + + const zipFs = new ZipFS(archive); + await zipFs.writeFilePromise(`/foo.txt` as PortablePath, `foo`); + + await xfs.unlinkPromise(archive); + + zipFs.saveAndClose(); + + expect((await xfs.statPromise(archive)).mode & 0o777).toStrictEqual(0o754); + }); + + ifNotWin32It(`should preserve the umask if the archive is unlinked before being closed (empty archive)`, async () => { + const tmpdir = xfs.mktempSync(); + const archive = `${tmpdir}/archive.zip` as PortablePath; + + await xfs.writeFilePromise(archive, makeEmptyArchive(), {mode: 0o754}); + + const zipFs = new ZipFS(archive); + + await xfs.unlinkPromise(archive); + + zipFs.saveAndClose(); + + expect((await xfs.statPromise(archive)).mode & 0o777).toStrictEqual(0o754); + }); + + ifNotWin32It(`should create archives with -rw-r--r--`, async () => { + const tmpdir = xfs.mktempSync(); + const archive = `${tmpdir}/archive.zip` as PortablePath; + + + const zipFs = new ZipFS(archive, {create: true}); + await zipFs.writeFilePromise(`/foo.txt` as PortablePath, `foo`); + + zipFs.saveAndClose(); + + expect((await xfs.statPromise(archive)).mode).toStrictEqual(S_IFREG | 0o644); + }); + + ifNotWin32It(`should create archives with -rw-r--r-- (empty archive)`, async () => { + const tmpdir = xfs.mktempSync(); + const archive = `${tmpdir}/archive.zip` as PortablePath; + + const zipFs = new ZipFS(archive, {create: true}); + + zipFs.saveAndClose(); + + expect((await xfs.statPromise(archive)).mode).toStrictEqual(S_IFREG | 0o644); + }); + + it(`should support chmod`, async () => { + const zipFs = new ZipFS(); + + zipFs.writeFileSync(`/foo.txt` as Filename, `foo`); + zipFs.chmodSync(`/foo.txt` as Filename, 0o754); + expect(zipFs.statSync(`/foo.txt` as Filename).mode & 0o777).toBe(0o754); + + await zipFs.writeFilePromise(`/bar.txt` as Filename, `bar`); + await zipFs.chmodPromise(`/bar.txt` as Filename, 0o754); + expect((await zipFs.statPromise(`/bar.txt` as Filename)).mode & 0o777).toBe(0o754); + + zipFs.discardAndClose(); + }); + + it(`should support fchmodSync`, () => { + const zipFs = new ZipFS(); + + zipFs.writeFileSync(`/foo.txt` as Filename, `foo`); + const fd = zipFs.openSync(`/foo.txt` as Filename, `rw`); + zipFs.fchmodSync(fd, 0o754); + zipFs.closeSync(fd); + expect(zipFs.statSync(`/foo.txt` as Filename).mode & 0o777).toBe(0o754); + + zipFs.discardAndClose(); + }); + + it(`should support writeFile mode`, async () => { + const zipFs = new ZipFS(); + + zipFs.writeFileSync(`/foo.txt` as Filename, `foo`, {mode: 0o754}); + expect(zipFs.statSync(`/foo.txt` as Filename).mode & 0o777).toBe(0o754); + + await zipFs.writeFilePromise(`/bar.txt` as Filename, `bar`, {mode: 0o754}); + expect((await zipFs.statPromise(`/bar.txt` as Filename)).mode & 0o777).toBe(0o754); + + zipFs.discardAndClose(); + }); + + it(`should support appendFile mode`, async () => { + const zipFs = new ZipFS(); + + zipFs.appendFileSync(`/foo.txt` as Filename, `foo`, {mode: 0o754}); + expect(zipFs.statSync(`/foo.txt` as Filename).mode & 0o777).toBe(0o754); + + await zipFs.appendFilePromise(`/bar.txt` as Filename, `bar`, {mode: 0o754}); + expect((await zipFs.statPromise(`/bar.txt` as Filename)).mode & 0o777).toBe(0o754); + + zipFs.discardAndClose(); + }); + + it(`should support mkdir mode`, async () => { + const zipFs = new ZipFS(); + + zipFs.mkdirSync(`/foo` as Filename, {mode: 0o754}); + expect(zipFs.statSync(`/foo` as Filename).mode & 0o777).toBe(0o754); + + await zipFs.mkdirPromise(`/bar` as Filename, {mode: 0o754}); + expect((await zipFs.statPromise(`/bar` as Filename)).mode & 0o777).toBe(0o754); + + zipFs.discardAndClose(); + }); + + it(`should support fd in writeFile and readFile`, async () => { + const zipFs = new ZipFS(); + + zipFs.mkdirSync(`/dir` as PortablePath); + zipFs.writeFileSync(`/dir/file` as PortablePath, `file content`); + + const fd = zipFs.openSync(`/dir/file` as PortablePath, `r`); + zipFs.writeFileSync(fd, `new content`); + + await expect(zipFs.readFilePromise(fd, `utf8`)).resolves.toEqual(`new content`); + + await zipFs.writeFilePromise(fd, `new new content`); + + expect(zipFs.readFileSync(fd, `utf8`)).toEqual(`new new content`); + + zipFs.discardAndClose(); + }); + + it(`should throw ENOTDIR when trying to stat a file as a directory`, () => { + const zipFs = new ZipFS(); + + zipFs.writeFileSync(`/foo.txt` as PortablePath, ``); + expect(() => zipFs.statSync(`/foo.txt/` as PortablePath)).toThrowError(`ENOTDIR`); + + zipFs.symlinkSync(`/foo.txt` as PortablePath, `/bar.txt` as PortablePath); + expect(() => zipFs.lstatSync(`/bar.txt/` as PortablePath)).toThrowError(`ENOTDIR`); + + zipFs.discardAndClose(); + }); + + it(`should throw ENOTDIR when trying to create a file when the dirname is a file`, () => { + const zipFs = new ZipFS(); + + zipFs.writeFileSync(`/foo.txt` as PortablePath, ``); + expect(() => zipFs.writeFileSync(`/foo.txt/bar.txt` as PortablePath, ``)).toThrowError(`ENOTDIR`); + + zipFs.symlinkSync(`/foo.txt` as PortablePath, `/bar.txt` as PortablePath); + expect(() => zipFs.writeFileSync(`/bar.txt/baz.txt` as PortablePath, ``)).toThrowError(`ENOTDIR`); + + zipFs.discardAndClose(); + }); + + it(`should throw ENOENT when reading a file that doesn't exist`, () => { + const zipFs = new ZipFS(); + + // File doesn't exist + expect(() => zipFs.readFileSync(`/foo` as PortablePath)).toThrowError(`ENOENT`); + + // Parent entry doesn't exist + expect(() => zipFs.readFileSync(`/foo/bar` as PortablePath)).toThrowError(`ENOENT`); + + zipFs.discardAndClose(); + }); + + it(`should return the first created directory in mkdir recursive`, () => { + const zipFs = new ZipFS(); + + expect(zipFs.mkdirSync(`/foo` as PortablePath, {recursive: true})).toEqual(`/foo` as PortablePath); + expect(zipFs.mkdirSync(`/foo` as PortablePath, {recursive: true})).toEqual(undefined); + expect(zipFs.mkdirSync(`/foo/bar/baz` as PortablePath, {recursive: true})).toEqual(`/foo/bar` as PortablePath); + expect(zipFs.mkdirSync(`/foo/bar/baz` as PortablePath, {recursive: true})).toEqual(undefined); + + zipFs.discardAndClose(); + }); + + it(`should return the first created directory in mkdirp`, () => { + const zipFs = new ZipFS(); + + expect(zipFs.mkdirpSync(`/foo` as PortablePath)).toEqual(`/foo` as PortablePath); + expect(zipFs.mkdirpSync(`/foo` as PortablePath)).toEqual(undefined); + expect(zipFs.mkdirpSync(`/foo/bar/baz` as PortablePath)).toEqual(`/foo/bar` as PortablePath); + expect(zipFs.mkdirpSync(`/foo/bar/baz` as PortablePath)).toEqual(undefined); + + zipFs.discardAndClose(); + }); + + it(`should support the recursive flag in readdir`, () => { + const zipFs = new ZipFS(); + + zipFs.mkdirSync(`/foo` as PortablePath); + zipFs.mkdirSync(`/foo/bar` as PortablePath); + zipFs.mkdirSync(`/bar` as PortablePath); + + zipFs.writeFileSync(`/foo/file.txt` as PortablePath, `Test`); + zipFs.writeFileSync(`/foo/bar/file.txt` as PortablePath, `Test`); + zipFs.writeFileSync(`/bar/file.txt` as PortablePath, `Test`); + zipFs.writeFileSync(`/file.txt` as PortablePath, `Test`); + + const zipContent = zipFs.getBufferAndClose(); + + const zipFs2 = new ZipFS(zipContent); + expect(zipFs2.readdirSync(`/` as PortablePath, {recursive: true}).sort()).toEqual([ + `bar`, + `bar/file.txt`, + `file.txt`, + `foo`, + `foo/bar`, + `foo/bar/file.txt`, + `foo/file.txt`, + ]); + + expect(zipFs2.readdirSync(`/foo` as PortablePath, {recursive: true}).sort()).toEqual([ + `bar`, + `bar/file.txt`, + `file.txt`, + ]); + }); + + it(`should support the combination of recursive and withFileTypes in readdir`, () => { + const zipFs = new ZipFS(); + + zipFs.mkdirSync(`/foo` as PortablePath); + zipFs.mkdirSync(`/foo/bar` as PortablePath); + zipFs.mkdirSync(`/bar` as PortablePath); + + zipFs.writeFileSync(`/foo/file.txt` as PortablePath, `Test`); + zipFs.writeFileSync(`/foo/bar/file.txt` as PortablePath, `Test`); + zipFs.writeFileSync(`/bar/file.txt` as PortablePath, `Test`); + zipFs.writeFileSync(`/file.txt` as PortablePath, `Test`); + + const zipContent = zipFs.getBufferAndClose(); + + const readdir = (p: PortablePath) => { + return zipFs2.readdirSync(p, {recursive: true, withFileTypes: true}).sort((a, b) => { + return a.path.localeCompare(b.path) || a.name.localeCompare(b.name); + }).map(({name, path}) => { + return {name, path}; + }); + }; + + const zipFs2 = new ZipFS(zipContent); + expect(readdir(PortablePath.root)).toEqual([ + {name: `bar`, path: `.`}, + {name: `file.txt`, path: `.`}, + {name: `foo`, path: `.`}, + {name: `file.txt`, path: `bar`}, + {name: `bar`, path: `foo`}, + {name: `file.txt`, path: `foo`}, + {name: `file.txt`, path: `foo/bar`}, + ]); + + expect(readdir(ppath.join(PortablePath.root, `foo`))).toEqual([ + {name: `bar`, path: `.`}, + {name: `file.txt`, path: `.`}, + {name: `file.txt`, path: `bar`}, + ]); + }); + + it(`should support throwIfNoEntry`, async () => { + const zipFs = new ZipFS(); + + expect(zipFs.statSync(`/foo` as PortablePath, {throwIfNoEntry: false})).toEqual(undefined); + expect(zipFs.statSync(`/foo/bar` as PortablePath, {throwIfNoEntry: false})).toEqual(undefined); + + expect(zipFs.lstatSync(`/foo` as PortablePath, {throwIfNoEntry: false})).toEqual(undefined); + expect(zipFs.lstatSync(`/foo/bar` as PortablePath, {throwIfNoEntry: false})).toEqual(undefined); + + await expect( + zipFs.statPromise(`/foo` as PortablePath, { + // @ts-expect-error throwIfNoEntry is not a valid option but statPromise + // calls statSync which does support it, this checks that it's ignored. + throwIfNoEntry: false, + }), + ).rejects.toMatchObject({ + code: `ENOENT`, + }); + + await expect( + zipFs.lstatPromise(`/foo` as PortablePath, { + // @ts-expect-error throwIfNoEntry is not a valid option but statPromise + // calls statSync which does support it, this checks that it's ignored. + throwIfNoEntry: false, + }), + ).rejects.toMatchObject({ + code: `ENOENT`, + }); + }); + + // https://github.com/nih-at/libzip/issues/146 + // https://github.com/yarnpkg/berry/pull/647 + // https://github.com/arcanis/libzip/commit/5f6dc0f43f23d4dd143f504270bb9c5de34c80a7 + it(`should be able to update the mtime after adding a file`, () => { + const zipFs = new ZipFS(); + zipFs.writeFileSync(`/foo.txt` as PortablePath, ``); + zipFs.utimesSync(`/foo.txt` as PortablePath, constants.SAFE_TIME, constants.SAFE_TIME); + + expect(zipFs.statSync(`/foo.txt` as PortablePath).mtimeMs).toEqual(constants.SAFE_TIME * 1000); + + zipFs.discardAndClose(); + }); +}); diff --git a/packages/yarnpkg-minizip/tests/ZipOpenFS.test.ts b/packages/yarnpkg-minizip/tests/ZipOpenFS.test.ts new file mode 100644 index 000000000000..a1e7e3386dc3 --- /dev/null +++ b/packages/yarnpkg-minizip/tests/ZipOpenFS.test.ts @@ -0,0 +1,249 @@ +import {ppath, npath, Filename, PortablePath} from '@yarnpkg/fslib'; +import {ZipOpenFS, getArchivePart} from '@yarnpkg/libzip'; + +export const ZIP_DIR1 = ppath.join( + npath.toPortablePath(__dirname), + `fixtures/foo.zip` as Filename, +); +export const ZIP_DIR2 = ppath.join( + npath.toPortablePath(__dirname), + `fixtures/folder.zip/foo.zip` as Filename, +); +export const ZIP_DIR3 = ppath.join( + npath.toPortablePath(__dirname), + `fixtures/foo.hiddenzip` as Filename, +); +export const ZIP_DIR4 = ppath.join( + npath.toPortablePath(__dirname), + `fixtures/symlink.zip` as Filename, +); + +export const ZIP_FILE1 = ppath.join(ZIP_DIR1, `foo.txt`); +export const ZIP_FILE2 = ppath.join(ZIP_DIR2, `foo.txt`); +export const ZIP_FILE3 = ppath.join(ZIP_DIR3, `foo.txt`); +export const ZIP_FILE4 = ppath.join(ZIP_DIR4, `foo.txt`); + +afterEach(() => { + jest.useRealTimers(); +}); + +describe(`getArchivePart`, () => { + const tests = [ + [`.zip`, null], + [`foo`, null], + [`foo.zip`, `foo.zip`], + [`foo.zip/bar`, `foo.zip`], + [`foo.zip/bar/baz`, `foo.zip`], + [`/a/b/c/foo.zip`, `/a/b/c/foo.zip`], + [`./a/b/c/foo.zip`, `./a/b/c/foo.zip`], + [`./a/b/c/.zip`, null], + [`./a/b/c/foo.zipp`, null], + [`./a/b/c/foo.zip/bar/baz/qux.zip`, `./a/b/c/foo.zip`], + [`./a/b/c/foo.zip-bar.zip`, `./a/b/c/foo.zip-bar.zip`], + [`./a/b/c/foo.zip-bar.zip/bar/baz/qux.zip`, `./a/b/c/foo.zip-bar.zip`], + [`./a/b/c/foo.zip-bar/foo.zip-bar/foo.zip-bar.zip/d`, `./a/b/c/foo.zip-bar/foo.zip-bar/foo.zip-bar.zip`], + ] as const; + + for (const [path, result] of tests) { + test(`getArchivePart(${JSON.stringify(path)}) === ${JSON.stringify(result)}`, () => { + expect(getArchivePart(path, `.zip`)).toStrictEqual(result); + }); + } +}); + +describe(`ZipOpenFS`, () => { + it(`can read from a zip file`, () => { + const fs = new ZipOpenFS(); + + expect(fs.readFileSync(ZIP_FILE1, `utf8`)).toEqual(`foo\n`); + + fs.discardAndClose(); + }); + + it(`can read from a zip file in a path containing .zip`, () => { + const fs = new ZipOpenFS(); + + expect(fs.readFileSync(ZIP_FILE2, `utf8`)).toEqual(`foo\n`); + + fs.discardAndClose(); + }); + + it(`can read from a zip file with an unusual extension if so configured`, () => { + const fs = new ZipOpenFS({fileExtensions: [`.hiddenzip`]}); + + expect(fs.readFileSync(ZIP_FILE3, `utf8`)).toEqual(`foo\n`); + + fs.discardAndClose(); + }); + + it(`throws when reading from a zip file with an unusual extension`, () => { + const fs = new ZipOpenFS(); + + expect(() => { + fs.readFileSync(ZIP_FILE3, `utf8`); + }).toThrowError(); + + fs.discardAndClose(); + }); + + it(`can read from a zip file that's a symlink`, () => { + const fs = new ZipOpenFS(); + + expect(fs.readFileSync(ZIP_FILE4, `utf8`)).toEqual(`foo\n`); + + fs.discardAndClose(); + }); + + it(`doesn't close a ZipFS instance with open handles`, () => { + const fs = new ZipOpenFS({maxOpenFiles: 1}); + + const fileHandle = fs.openSync(ZIP_FILE1, ``); + + expect(fs.readFileSync(ZIP_FILE2, `utf8`)).toEqual(`foo\n`); + + const buff = Buffer.alloc(4); + fs.readSync(fileHandle, buff, 0, 4, 0); + fs.closeSync(fileHandle); + + expect(buff.toString(`utf8`)).toEqual(`foo\n`); + + fs.discardAndClose(); + }); + + it(`sets the path property of the stream object returned by createReadStream to the normalized native version of the input path`, async () => { + const fs = new ZipOpenFS({maxOpenFiles: 1}); + + const unnormalizedPortablePath = ZIP_FILE1.replace(/\//g, `/./`) as PortablePath; + const normalizedNativePath = npath.fromPortablePath(ZIP_FILE1); + + const stream = fs.createReadStream(unnormalizedPortablePath); + + expect(stream.path).toMatch(normalizedNativePath); + + stream.destroy(); + fs.discardAndClose(); + }); + + it(`treats createReadStream as an open file handle`, async () => { + const fs = new ZipOpenFS({maxOpenFiles: 1}); + + const chunks: Array = []; + await new Promise(resolve => { + let done = 0; + + fs.createReadStream(ZIP_FILE1) + .on(`data`, (chunk: Buffer) => { + chunks.push(chunk); + }) + .on(`close`, () => { + if (++done === 2) { + resolve(); + } + }); + + fs.createReadStream(ZIP_FILE2) + .on(`data`, (chunk: Buffer) => { + chunks.push(chunk); + }) + .on(`close`, () => { + if (++done === 2) { + resolve(); + } + }); + }); + + expect(chunks[0].toString(`utf8`)).toMatch(`foo\n`); + expect(chunks[1].toString(`utf8`)).toMatch(`foo\n`); + + fs.discardAndClose(); + }); + + it(`treats createWriteStream as an open file handle`, async () => { + const fs = new ZipOpenFS({maxOpenFiles: 1}); + + const stream1 = fs.createWriteStream(ZIP_FILE1); + const stream2 = fs.createWriteStream(ZIP_FILE2); + + await new Promise(resolve => { + let done = 0; + stream1.end(`foo`, () => { + if (++done === 2) { + resolve(); + } + }); + stream2.end(`bar`, () => { + if (++done === 2) { + resolve(); + } + }); + }); + + fs.discardAndClose(); + }); + + it(`closes ZipFS instances once they become stale`, async () => { + jest.useFakeTimers(); + + const fs = new ZipOpenFS({maxAge: 2000}); + + await fs.existsPromise(ZIP_FILE1); + // @ts-expect-error: mountInstances is private + expect(fs.mountInstances!.size).toEqual(1); + + jest.advanceTimersByTime(1000); + + fs.existsSync(ZIP_FILE2); + // @ts-expect-error: mountInstances is private + expect(fs.mountInstances!.size).toEqual(2); + + jest.advanceTimersByTime(1000); + + // @ts-expect-error: mountInstances is private + expect(fs.mountInstances!.size).toEqual(1); + + jest.advanceTimersByTime(1000); + + // @ts-expect-error: mountInstances is private + expect(fs.mountInstances!.size).toEqual(0); + + fs.discardAndClose(); + }); + + it(`doesn't close zip files while they are in use`, async () => { + const fs = new ZipOpenFS({maxOpenFiles: 1}); + + await Promise.all([ + fs.readFilePromise(ZIP_FILE1), + fs.realpathPromise(ZIP_FILE1), + fs.readFilePromise(ZIP_FILE2), + fs.realpathPromise(ZIP_FILE2), + ]); + + fs.discardAndClose(); + }); + + it(`doesn't crash when watching a file in a archive that gets closed`, async () => { + jest.useFakeTimers(); + + const fs = new ZipOpenFS({maxOpenFiles: 1}); + + fs.watchFile(ZIP_FILE1, (current, previous) => {}); + fs.watchFile(ZIP_FILE2, (current, previous) => {}); + + jest.advanceTimersByTime(100); + + fs.discardAndClose(); + }); + + it(`treats Dir instances opened via opendir as open file handles`, () => { + const fs = new ZipOpenFS({maxOpenFiles: 1}); + + const dir1 = fs.opendirSync(ZIP_DIR1); + const dir2 = fs.opendirSync(ZIP_DIR2); + + expect(dir1.readSync()!.name).toStrictEqual(`foo.txt`); + expect(dir2.readSync()!.name).toStrictEqual(`foo.txt`); + + fs.discardAndClose(); + }); +}); diff --git a/packages/yarnpkg-minizip/tests/fixtures/folder.zip/foo.zip b/packages/yarnpkg-minizip/tests/fixtures/folder.zip/foo.zip new file mode 100644 index 000000000000..b93c88559ed3 Binary files /dev/null and b/packages/yarnpkg-minizip/tests/fixtures/folder.zip/foo.zip differ diff --git a/packages/yarnpkg-minizip/tests/fixtures/foo.hiddenzip b/packages/yarnpkg-minizip/tests/fixtures/foo.hiddenzip new file mode 100644 index 000000000000..b93c88559ed3 Binary files /dev/null and b/packages/yarnpkg-minizip/tests/fixtures/foo.hiddenzip differ diff --git a/packages/yarnpkg-minizip/tests/fixtures/foo.zip b/packages/yarnpkg-minizip/tests/fixtures/foo.zip new file mode 100644 index 000000000000..b93c88559ed3 Binary files /dev/null and b/packages/yarnpkg-minizip/tests/fixtures/foo.zip differ diff --git a/packages/yarnpkg-minizip/tests/fixtures/symlink-source.zip b/packages/yarnpkg-minizip/tests/fixtures/symlink-source.zip new file mode 100644 index 000000000000..b93c88559ed3 Binary files /dev/null and b/packages/yarnpkg-minizip/tests/fixtures/symlink-source.zip differ diff --git a/packages/yarnpkg-minizip/tests/fixtures/symlink.zip b/packages/yarnpkg-minizip/tests/fixtures/symlink.zip new file mode 120000 index 000000000000..75a9ac7bd2a5 --- /dev/null +++ b/packages/yarnpkg-minizip/tests/fixtures/symlink.zip @@ -0,0 +1 @@ +symlink-source.zip \ No newline at end of file diff --git a/packages/yarnpkg-pnp/package.json b/packages/yarnpkg-pnp/package.json index d37513d7165e..8d505cdd6041 100644 --- a/packages/yarnpkg-pnp/package.json +++ b/packages/yarnpkg-pnp/package.json @@ -17,6 +17,7 @@ "@rollup/plugin-node-resolve": "^11.0.1", "@types/semver": "^7.1.0", "@yarnpkg/libzip": "workspace:^", + "@yarnpkg/minizip": "workspace:^", "arg": "^5.0.2", "esbuild": "npm:esbuild-wasm@^0.23.0", "rollup": "^2.59.0", diff --git a/packages/yarnpkg-pnp/sources/esm-loader/fspatch.ts b/packages/yarnpkg-pnp/sources/esm-loader/fspatch.ts index ab6b2796ad94..c2d14c0ec6d2 100644 --- a/packages/yarnpkg-pnp/sources/esm-loader/fspatch.ts +++ b/packages/yarnpkg-pnp/sources/esm-loader/fspatch.ts @@ -54,7 +54,7 @@ if (!HAS_LAZY_LOADED_TRANSLATORS) { }; const originalfstat = binding.fstat; - // Those values must be synced with packages/yarnpkg-fslib/sources/ZipOpenFS.ts + // Those values must be synced with packages/yarnpkg-fslib/sources/ZipOpenFS.ts TODO???????? const ZIP_MASK = 0xff000000; const ZIP_MAGIC = 0x2a000000; diff --git a/packages/yarnpkg-pnp/sources/generateSerializedState.ts b/packages/yarnpkg-pnp/sources/generateSerializedState.ts index 081cadb4a9ed..64c96efa2bd4 100644 --- a/packages/yarnpkg-pnp/sources/generateSerializedState.ts +++ b/packages/yarnpkg-pnp/sources/generateSerializedState.ts @@ -107,7 +107,7 @@ export function generateSerializedState(settings: PnpSettings): SerializedState dependencyTreeRoots: settings.dependencyTreeRoots, enableTopLevelFallback: settings.enableTopLevelFallback || false, ignorePatternData: settings.ignorePattern || null, - + zipImplementation: settings.zipImplementation, fallbackExclusionList: generateFallbackExclusionList(settings), fallbackPool: generateFallbackPoolData(settings), packageRegistryData: generatePackageRegistryData(settings), diff --git a/packages/yarnpkg-pnp/sources/loader/_entryPoint.ts b/packages/yarnpkg-pnp/sources/loader/_entryPoint.ts index 6a423b1cdd06..3d8a141ac9b1 100644 --- a/packages/yarnpkg-pnp/sources/loader/_entryPoint.ts +++ b/packages/yarnpkg-pnp/sources/loader/_entryPoint.ts @@ -1,5 +1,6 @@ import {FakeFS, NodeFS, NativePath, PortablePath, VirtualFS, ProxiedFS, ppath} from '@yarnpkg/fslib'; import {ZipOpenFS} from '@yarnpkg/libzip'; +import {MiniZipOpenFS} from '@yarnpkg/minizip'; import fs from 'fs'; import Module from 'module'; import StringDecoder from 'string_decoder'; @@ -20,14 +21,17 @@ const localFs: typeof fs = {...fs}; const nodeFs = new NodeFS(localFs); const defaultRuntimeState = $$SETUP_STATE(hydrateRuntimeState); + const defaultPnpapiResolution = __filename; // We create a virtual filesystem that will do three things: // 1. all requests inside a folder named "__virtual___" will be remapped according the virtual folder rules // 2. all requests going inside a Zip archive will be handled by the Zip fs implementation // 3. any remaining request will be forwarded to Node as-is +const zipFSImpl = defaultRuntimeState.zipImplementation === 'minizip' ? MiniZipOpenFS : ZipOpenFS; + const defaultFsLayer: FakeFS = new VirtualFS({ - baseFs: new ZipOpenFS({ + baseFs: new zipFSImpl({ baseFs: nodeFs, maxOpenFiles: 80, readOnlyArchives: true, diff --git a/packages/yarnpkg-pnp/sources/loader/hydrateRuntimeState.ts b/packages/yarnpkg-pnp/sources/loader/hydrateRuntimeState.ts index fdcf2798c499..42361ec81080 100644 --- a/packages/yarnpkg-pnp/sources/loader/hydrateRuntimeState.ts +++ b/packages/yarnpkg-pnp/sources/loader/hydrateRuntimeState.ts @@ -68,6 +68,7 @@ export function hydrateRuntimeState(data: SerializedState, {basePath}: HydrateRu dependencyTreeRoots, enableTopLevelFallback, fallbackExclusionList, + zipImplementation: data.zipImplementation, fallbackPool, ignorePattern, packageLocatorsByLocations, diff --git a/packages/yarnpkg-pnp/sources/types.ts b/packages/yarnpkg-pnp/sources/types.ts index 2ef3d6f0ca98..3eefbc62d601 100644 --- a/packages/yarnpkg-pnp/sources/types.ts +++ b/packages/yarnpkg-pnp/sources/types.ts @@ -34,6 +34,8 @@ export type PackageRegistryData = Array<[string | null, PackageStoreData]>; export type LocationLengthData = Array; +export type ZipImplementation = 'libzip' | 'minizip'; + // This is what is stored within the .pnp.data.json file export type SerializedState = { // @eslint-ignore-next-line @typescript-eslint/naming-convention @@ -41,6 +43,7 @@ export type SerializedState = { enableTopLevelFallback: boolean; fallbackExclusionList: Array<[string, Array]>; fallbackPool: Array<[string, DependencyTarget]>; + zipImplementation: ZipImplementation ignorePatternData: string | null; packageRegistryData: PackageRegistryData; dependencyTreeRoots: Array; @@ -52,6 +55,7 @@ export type RuntimeState = { enableTopLevelFallback: boolean; fallbackExclusionList: Map>; fallbackPool: Map; + zipImplementation: ZipImplementation; ignorePattern: RegExp | null; packageLocatorsByLocations: Map; packageRegistry: PackageRegistry; @@ -86,6 +90,8 @@ export type PnpSettings = { // getDependencyTreeRoots function. They are typically the workspace // locators. dependencyTreeRoots: Array; + + zipImplementation: ZipImplementation }; export type ResolveToUnqualifiedOptions = { diff --git a/yarn.lock b/yarn.lock index 715c4570f200..366f07672007 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5715,6 +5715,19 @@ __metadata: languageName: unknown linkType: soft +"@yarnpkg/minizip@workspace:^, @yarnpkg/minizip@workspace:packages/yarnpkg-minizip": + version: 0.0.0-use.local + resolution: "@yarnpkg/minizip@workspace:packages/yarnpkg-minizip" + dependencies: + "@types/prettier": "npm:1.19.0" + "@yarnpkg/fslib": "workspace:^" + prettier: "npm:^1.19.1" + tslib: "npm:^2.4.0" + peerDependencies: + "@yarnpkg/fslib": "workspace:^" + languageName: unknown + linkType: soft + "@yarnpkg/monorepo@workspace:., @yarnpkg/monorepo@workspace:^": version: 0.0.0-use.local resolution: "@yarnpkg/monorepo@workspace:." @@ -5731,10 +5744,12 @@ __metadata: "@yarnpkg/eslint-config": "workspace:^" "@yarnpkg/fslib": "workspace:^" "@yarnpkg/libzip": "workspace:^" + "@yarnpkg/minizip": "workspace:^" "@yarnpkg/sdks": "workspace:^" "@yarnpkg/types": "workspace:^" chalk: "npm:^3.0.0" clipanion: "npm:^4.0.0-rc.2" + enhanced-resolve: "npm:^5.18.0" esbuild: "npm:esbuild-wasm@^0.23.0" eslint: "npm:^8.57.0" jest: "npm:^29.2.1" @@ -6222,6 +6237,7 @@ __metadata: "@types/semver": "npm:^7.1.0" "@yarnpkg/fslib": "workspace:^" "@yarnpkg/libzip": "workspace:^" + "@yarnpkg/minizip": "workspace:^" arg: "npm:^5.0.2" esbuild: "npm:esbuild-wasm@^0.23.0" rollup: "npm:^2.59.0" @@ -9426,6 +9442,16 @@ __metadata: languageName: node linkType: hard +"enhanced-resolve@npm:^5.18.0": + version: 5.18.0 + resolution: "enhanced-resolve@npm:5.18.0" + dependencies: + graceful-fs: "npm:^4.2.4" + tapable: "npm:^2.2.0" + checksum: 10/e88463ef97b68d40d0da0cd0c572e23f43dba0be622d6d44eae5cafed05f0c5dac43e463a83a86c4f70186d029357f82b56d9e1e47e8fc91dce3d6602f8bd6ce + languageName: node + linkType: hard + "enquirer@npm:^2.3.6": version: 2.3.6 resolution: "enquirer@npm:2.3.6"