[POC] Improve pnp loader speed and memory

bench/run-resolve.ts

@@ -0,0 +1,36 @@
+
+import * as fs from 'fs'
+import { performance } from 'perf_hooks';
+import { ZipFS } from '@yarnpkg/libzip';
+import { MiniZipFS } from '@yarnpkg/minizip';
+import { PortablePath } from '@yarnpkg/fslib';
+import {createRequire} from 'module'
+import path from 'path'
+import {ResolverFactory} from 'enhanced-resolve'
+
+
+
+const formatMemoryUsage = (data: number) => `${Math.round(data / 1024 / 1024 * 100) / 100} MB`;
+
+const from = require.resolve('micromatch');
+// const baseFs= new ZipFS(from as PortablePath)
+// const resolver = ResolverFactory.createResolver({fileSystem: {readFile: baseFs.readFileSync.bind(baseFs)}})
+console.log(from)
+const req = createRequire(path.dirname(from))
+
+globalThis.Error = class {
+  constructor() {}
+  static captureStackTrace = () => {}
+}
+
+
+const was = performance.now()
+for (let i = 0; i < 50_000; i++) {
+  // resolver.resolveSync({}, from, `./fi${i}`)
+  try {
+    req.resolve(`./file${i}`)
+  } catch {
+
+  }
+}
+console.log(performance.now() - was)

bench/run.ts

@@ -0,0 +1,80 @@
+
+import * as fs from 'fs'
+import { performance } from 'perf_hooks';
+import { ZipFS } from '@yarnpkg/libzip';
+import { MiniZipFS } from '@yarnpkg/minizip';
+import { PortablePath } from '@yarnpkg/fslib';
+
+
+
+
+
+const cwd = '/Users/vadymh/.yarn/berry/cache';
+const files = fs.readdirSync(cwd).filter(f => f.endsWith('.zip'))
+
+let totalSize = 0
+for (const f of files) {
+  const file = `${cwd}/${f}`;
+  const stats = fs.statSync(file)
+  totalSize += stats.size
+}
+
+
+const formatMemoryUsage = (data: number) => `${Math.round(data / 1024 / 1024 * 100) / 100} MB`;
+
+console.log('files count: ' + files.length, 'size', formatMemoryUsage(totalSize))
+
+const mem = process.argv.includes('--mem')
+
+
+for (const zipper of [MiniZipFS, ZipFS]) {
+  for (let i = 0; i < 3; i++) {
+    let allFiles = 0
+    const was = performance.now()
+    const memArr = []
+    for (const f of files) {
+      const file = `${cwd}/${f}`;
+
+      let fi = new zipper(file as PortablePath)
+      allFiles += fi.getAllFiles().length
+      if (mem){ 
+        const memoryData = process.memoryUsage();
+        memArr.push(memoryData.rss)
+      }
+
+      fi.discardAndClose()
+    }
+
+    // const memoryUsage = {
+    //   rss: `${formatMemoryUsage(memoryData.rss)} -> Resident Set Size - total memory allocated for the process execution`,
+    //   heapTotal: `${formatMemoryUsage(memoryData.heapTotal)} -> total size of the allocated heap`,
+    //   heapUsed: `${formatMemoryUsage(memoryData.heapUsed)} -> actual memory used during the execution`,
+    //   external: `${formatMemoryUsage(memoryData.external)} -> V8 external memory`,
+    // };
+
+    console.log(`${zipper.name}: try ${i + 1}`)
+
+    if (mem) {
+      memArr.sort((a, b) => a-b)
+      console.log({
+        p50: formatMemoryUsage(memArr[Math.floor(memArr.length / 2)]),
+        p90: formatMemoryUsage(memArr[Math.floor(memArr.length * 0.9)]),
+        p95: formatMemoryUsage(memArr[Math.floor(memArr.length * 0.95)]),
+        p99: formatMemoryUsage(memArr[Math.floor(memArr.length * 0.99)]),
+      })
+    } else {
+      const took = performance.now() - was
+      console.log({
+        took,
+        allFiles
+      })
+    }
+
+    // console.log(memoryUsage)
+  }
+}
+
+
+
+
+

package.json

@@ -17,8 +17,10 @@
     "@yarnpkg/eslint-config": "workspace:^",
     "@yarnpkg/fslib": "workspace:^",
     "@yarnpkg/libzip": "workspace:^",
+    "@yarnpkg/minizip": "workspace:^",
     "@yarnpkg/sdks": "workspace:^",
     "clipanion": "^4.0.0-rc.2",
+    "enhanced-resolve": "^5.18.0",
     "esbuild": "npm:esbuild-wasm@^0.23.0",
     "eslint": "^8.57.0",
     "jest": "^29.2.1",

packages/plugin-pnp/sources/PnpLinker.ts

@@ -285,6 +285,7 @@ export class PnpInstaller implements Installer {
       fallbackExclusionList,
       fallbackPool,
       ignorePattern,
+      zipImplementation: this.opts.project.configuration.get(`minizip`) ? `minizip` : `libzip`,
       packageRegistry,
       shebang,
     });

packages/plugin-pnp/sources/index.ts

@@ -70,6 +70,7 @@ declare module '@yarnpkg/core' {
     nodeLinker: string;
     winLinkType: string;
     pnpMode: string;
+    minizip: boolean
     pnpShebang: string;
     pnpIgnorePatterns: Array<string>;
     pnpEnableEsmLoader: boolean;
@@ -90,6 +91,11 @@ const plugin: Plugin<CoreHooks & StageHooks> = {
       type: SettingsType.STRING,
       default: `pnp`,
     },
+    minizip: {
+      description: `Whether Yarn should use minizip to extract archives`,
+      type: SettingsType.BOOLEAN,
+      default: false
+    },
     winLinkType: {
       description: `Whether Yarn should use Windows Junctions or symlinks when creating links on Windows.`,
       type: SettingsType.STRING,

packages/yarnpkg-minizip/.babelrc.js

@@ -0,0 +1,5 @@
+module.exports = {
+  ignore: [
+    `./sources/libzip.js`,
+  ],
+};

packages/yarnpkg-minizip/README.md

@@ -0,0 +1,3 @@
+# `@yarnpkg/minizip`
+
+Lightweight, fast, readonly zip fs for pnp runtime.

packages/yarnpkg-minizip/package.json

@@ -0,0 +1,50 @@
+{
+  "name": "@yarnpkg/minizip",
+  "version": "3.1.0",
+  "license": "BSD-2-Clause",
+  "main": "./sources/index.ts",
+  "exports": {
+    ".": {
+      "default": "./sources/index.ts"
+    },
+    "./package.json": "./package.json"
+  },
+  "scripts": {
+    "postpack": "rm -rf lib",
+    "prepack": "run build:compile \"$(pwd)\"",
+    "release": "yarn npm publish"
+  },
+  "publishConfig": {
+    "main": "./lib/index.js",
+    "browser": "./lib/index.js",
+    "exports": {
+      ".": {
+        "default": "./lib/index.js"
+      },
+      "./package.json": "./package.json"
+    }
+  },
+  "files": [
+    "/lib/**/*"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "ssh://git@github.com/yarnpkg/berry.git",
+    "directory": "packages/yarnpkg-minizip"
+  },
+  "devDependencies": {
+    "@types/prettier": "1.19.0",
+    "prettier": "^1.19.1"
+  },
+  "dependencies": {
+    "@yarnpkg/fslib": "workspace:^",
+    "tslib": "^2.4.0"
+  },
+  "peerDependencies": {
+    "@yarnpkg/fslib": "workspace:^"
+  },
+  "engines": {
+    "node": ">=18.12.0"
+  },
+  "stableVersion": "3.1.0"
+}