From 91dd15ac7a5dff1116cc804c774e0bba2840e0bb Mon Sep 17 00:00:00 2001 From: Juraj Carnogursky Date: Thu, 20 Apr 2023 17:07:02 +0200 Subject: [PATCH 01/12] Replace dependency on broken node-unzipper with native zlib Fixes: https://github.com/firebase/firebase-tools/issues/5614 https://github.com/firebase/firebase-tools/issues/5677 https://github.com/firebase/firebase-tools-ui/issues/939 https://github.com/firebase/firebase-tools-ui/issues/940 --- npm-shrinkwrap.json | 298 ------------------ package.json | 2 - src/emulator/download.ts | 15 +- src/extensions/extensionsHelper.ts | 4 +- .../zip-files/node-unzipper-testData/LICENSE | 25 ++ .../compressed-cp866/archive.zip | Bin 0 -> 163 bytes .../\320\242\320\265\321\201\321\202.txt" | 1 + .../compressed-directory-entry/archive.zip | Bin 0 -> 2480 bytes .../inflated/META-INF/container.xml | 8 + .../inflated/content.opf | 22 ++ .../inflated/index.html | 15 + .../inflated/mimetype | 1 + .../inflated/page_styles.css | 4 + .../inflated/stylesheet.css | 11 + .../inflated/toc.ncx | 21 ++ .../compressed-flags-set/archive.zip | Bin 0 -> 1636 bytes .../inflated/dir/fileInsideDir.txt | 1 + .../compressed-flags-set/inflated/file.txt | 11 + .../compressed-standard/archive.zip | Bin 0 -> 1636 bytes .../compressed-standard/broken.zip | Bin 0 -> 1100 bytes .../inflated/dir/fileInsideDir.txt | 1 + .../compressed-standard/inflated/file.txt | 11 + .../uncompressed/archive.zip | Bin 0 -> 489 bytes .../inflated/dir/fileInsideDir.txt | 1 + .../uncompressed/inflated/file.txt | 1 + .../zip-slip/archive.zip | Bin 0 -> 545 bytes .../zip-slip/inflated/good.txt | 1 + .../node-unzipper-testData/zip64/archive.zip | Bin 0 -> 242 bytes .../zip64/inflated/README | 1 + src/test/unzip.spec.ts | 161 ++++++++++ src/unzip.ts | 121 +++++++ 31 files changed, 421 insertions(+), 316 deletions(-) create mode 100644 src/test/fixtures/zip-files/node-unzipper-testData/LICENSE create mode 100644 src/test/fixtures/zip-files/node-unzipper-testData/compressed-cp866/archive.zip create mode 100644 "src/test/fixtures/zip-files/node-unzipper-testData/compressed-cp866/inflated/\320\242\320\265\321\201\321\202.txt" create mode 100644 src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/archive.zip create mode 100644 src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/META-INF/container.xml create mode 100644 src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/content.opf create mode 100644 src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/index.html create mode 100644 src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/mimetype create mode 100644 src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/page_styles.css create mode 100644 src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/stylesheet.css create mode 100644 src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/toc.ncx create mode 100644 src/test/fixtures/zip-files/node-unzipper-testData/compressed-flags-set/archive.zip create mode 100644 src/test/fixtures/zip-files/node-unzipper-testData/compressed-flags-set/inflated/dir/fileInsideDir.txt create mode 100644 src/test/fixtures/zip-files/node-unzipper-testData/compressed-flags-set/inflated/file.txt create mode 100644 src/test/fixtures/zip-files/node-unzipper-testData/compressed-standard/archive.zip create mode 100644 src/test/fixtures/zip-files/node-unzipper-testData/compressed-standard/broken.zip create mode 100644 src/test/fixtures/zip-files/node-unzipper-testData/compressed-standard/inflated/dir/fileInsideDir.txt create mode 100644 src/test/fixtures/zip-files/node-unzipper-testData/compressed-standard/inflated/file.txt create mode 100644 src/test/fixtures/zip-files/node-unzipper-testData/uncompressed/archive.zip create mode 100644 src/test/fixtures/zip-files/node-unzipper-testData/uncompressed/inflated/dir/fileInsideDir.txt create mode 100644 src/test/fixtures/zip-files/node-unzipper-testData/uncompressed/inflated/file.txt create mode 100644 src/test/fixtures/zip-files/node-unzipper-testData/zip-slip/archive.zip create mode 100644 src/test/fixtures/zip-files/node-unzipper-testData/zip-slip/inflated/good.txt create mode 100644 src/test/fixtures/zip-files/node-unzipper-testData/zip64/archive.zip create mode 100644 src/test/fixtures/zip-files/node-unzipper-testData/zip64/inflated/README create mode 100644 src/test/unzip.spec.ts create mode 100644 src/unzip.ts diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index c22cade0c15..ff7eceb012c 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -64,7 +64,6 @@ "tmp": "^0.2.1", "triple-beam": "^1.3.0", "universal-analytics": "^0.5.3", - "unzipper": "^0.10.10", "update-notifier-cjs": "^5.1.6", "uuid": "^8.3.2", "winston": "^3.0.0", @@ -123,7 +122,6 @@ "@types/tmp": "^0.2.3", "@types/triple-beam": "^1.3.0", "@types/universal-analytics": "^0.4.5", - "@types/unzipper": "^0.10.0", "@types/update-notifier": "^5.1.0", "@types/uuid": "^8.3.1", "@types/ws": "^7.2.3", @@ -4094,15 +4092,6 @@ "integrity": "sha512-Opb+Un786PS3te24VtJR/QPmX00P/pXaJQtLQYJklQefP4xP0Ic3mPc2z6SDz97OrITzR+RHTBEwjtNRjZ/nLQ==", "dev": true }, - "node_modules/@types/unzipper": { - "version": "0.10.0", - "resolved": "https://registry.npmjs.org/@types/unzipper/-/unzipper-0.10.0.tgz", - "integrity": "sha512-GZL5vt0o9ZAST+7ge1Sirzc14EEJFbq6kib24nS0UglY6BHX8ERhA8cBq4XsYWcGK212FtMBZyJz6AwPvrhGLQ==", - "dev": true, - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/update-notifier": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/@types/update-notifier/-/update-notifier-5.1.0.tgz", @@ -5643,14 +5632,6 @@ "tweetnacl": "^0.14.3" } }, - "node_modules/big-integer": { - "version": "1.6.48", - "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.48.tgz", - "integrity": "sha512-j51egjPa7/i+RdiRuJbPdJ2FIUYYPhvYLjzoYbcMMm62ooO6F94fETG4MTs46zPAF9Brs04OajboA/qTGuz78w==", - "engines": { - "node": ">=0.6" - } - }, "node_modules/bignumber.js": { "version": "9.0.2", "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.2.tgz", @@ -5659,18 +5640,6 @@ "node": "*" } }, - "node_modules/binary": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", - "integrity": "sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=", - "dependencies": { - "buffers": "~0.1.1", - "chainsaw": "~0.1.0" - }, - "engines": { - "node": "*" - } - }, "node_modules/binary-extensions": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.0.0.tgz", @@ -5689,11 +5658,6 @@ "readable-stream": "^3.4.0" } }, - "node_modules/bluebird": { - "version": "3.4.7", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz", - "integrity": "sha1-9y12C+Cbf3bQjtj66Ysomo0F+rM=" - }, "node_modules/body-parser": { "version": "1.20.1", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", @@ -5922,22 +5886,6 @@ "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", "dev": true }, - "node_modules/buffer-indexof-polyfill": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.1.tgz", - "integrity": "sha1-qfuAbOgUXVQoUQznLyeLs2OmOL8=", - "engines": { - "node": ">=0.10" - } - }, - "node_modules/buffers": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", - "integrity": "sha1-skV5w77U1tOWru5tmorn9Ugqt7s=", - "engines": { - "node": ">=0.2.0" - } - }, "node_modules/busboy": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", @@ -6222,17 +6170,6 @@ "chai": ">= 2.1.2 < 5" } }, - "node_modules/chainsaw": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", - "integrity": "sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=", - "dependencies": { - "traverse": ">=0.3.0 <0.4" - }, - "engines": { - "node": "*" - } - }, "node_modules/chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -7198,28 +7135,6 @@ "node": ">=4" } }, - "node_modules/duplexer2": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", - "integrity": "sha1-ixLauHjA1p4+eJEFFmKjL8a93ME=", - "dependencies": { - "readable-stream": "^2.0.2" - } - }, - "node_modules/duplexer2/node_modules/readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, "node_modules/duplexify": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.1.tgz", @@ -8909,31 +8824,6 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, - "node_modules/fstream": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz", - "integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==", - "dependencies": { - "graceful-fs": "^4.1.2", - "inherits": "~2.0.0", - "mkdirp": ">=0.5 0", - "rimraf": "2" - }, - "engines": { - "node": ">=0.6" - } - }, - "node_modules/fstream/node_modules/rimraf": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", - "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", - "dependencies": { - "glob": "^7.1.3" - }, - "bin": { - "rimraf": "bin.js" - } - }, "node_modules/ftp": { "version": "0.3.10", "resolved": "https://registry.npmjs.org/ftp/-/ftp-0.3.10.tgz", @@ -11641,11 +11531,6 @@ "uc.micro": "^1.0.1" } }, - "node_modules/listenercount": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/listenercount/-/listenercount-1.0.1.tgz", - "integrity": "sha1-hMinKrWcRyUyFIDJdeZQg0LnCTc=" - }, "node_modules/load-yaml-file": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/load-yaml-file/-/load-yaml-file-0.2.0.tgz", @@ -16396,11 +16281,6 @@ "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", "devOptional": true }, - "node_modules/setimmediate": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", - "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=" - }, "node_modules/setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -17685,14 +17565,6 @@ "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=" }, - "node_modules/traverse": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", - "integrity": "sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk=", - "engines": { - "node": "*" - } - }, "node_modules/trim-lines": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/trim-lines/-/trim-lines-3.0.1.tgz", @@ -18241,37 +18113,6 @@ "node": ">= 0.8" } }, - "node_modules/unzipper": { - "version": "0.10.10", - "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.10.10.tgz", - "integrity": "sha512-wEgtqtrnJ/9zIBsQb8UIxOhAH1eTHfi7D/xvmrUoMEePeI6u24nq1wigazbIFtHt6ANYXdEVTvc8XYNlTurs7A==", - "dependencies": { - "big-integer": "^1.6.17", - "binary": "~0.3.0", - "bluebird": "~3.4.1", - "buffer-indexof-polyfill": "~1.0.0", - "duplexer2": "~0.1.4", - "fstream": "^1.0.12", - "graceful-fs": "^4.2.2", - "listenercount": "~1.0.1", - "readable-stream": "~2.3.6", - "setimmediate": "~1.0.4" - } - }, - "node_modules/unzipper/node_modules/readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, "node_modules/update-browserslist-db": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.9.tgz", @@ -22423,15 +22264,6 @@ "integrity": "sha512-Opb+Un786PS3te24VtJR/QPmX00P/pXaJQtLQYJklQefP4xP0Ic3mPc2z6SDz97OrITzR+RHTBEwjtNRjZ/nLQ==", "dev": true }, - "@types/unzipper": { - "version": "0.10.0", - "resolved": "https://registry.npmjs.org/@types/unzipper/-/unzipper-0.10.0.tgz", - "integrity": "sha512-GZL5vt0o9ZAST+7ge1Sirzc14EEJFbq6kib24nS0UglY6BHX8ERhA8cBq4XsYWcGK212FtMBZyJz6AwPvrhGLQ==", - "dev": true, - "requires": { - "@types/node": "*" - } - }, "@types/update-notifier": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/@types/update-notifier/-/update-notifier-5.1.0.tgz", @@ -23540,25 +23372,11 @@ "tweetnacl": "^0.14.3" } }, - "big-integer": { - "version": "1.6.48", - "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.48.tgz", - "integrity": "sha512-j51egjPa7/i+RdiRuJbPdJ2FIUYYPhvYLjzoYbcMMm62ooO6F94fETG4MTs46zPAF9Brs04OajboA/qTGuz78w==" - }, "bignumber.js": { "version": "9.0.2", "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.2.tgz", "integrity": "sha512-GAcQvbpsM0pUb0zw1EI0KhQEZ+lRwR5fYaAp3vPOYuP7aDvGy6cVN6XHLauvF8SOga2y0dcLcjt3iQDTSEliyw==" }, - "binary": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", - "integrity": "sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=", - "requires": { - "buffers": "~0.1.1", - "chainsaw": "~0.1.0" - } - }, "binary-extensions": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.0.0.tgz", @@ -23574,11 +23392,6 @@ "readable-stream": "^3.4.0" } }, - "bluebird": { - "version": "3.4.7", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz", - "integrity": "sha1-9y12C+Cbf3bQjtj66Ysomo0F+rM=" - }, "body-parser": { "version": "1.20.1", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", @@ -23741,16 +23554,6 @@ "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", "dev": true }, - "buffer-indexof-polyfill": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.1.tgz", - "integrity": "sha1-qfuAbOgUXVQoUQznLyeLs2OmOL8=" - }, - "buffers": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", - "integrity": "sha1-skV5w77U1tOWru5tmorn9Ugqt7s=" - }, "busboy": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", @@ -23957,14 +23760,6 @@ "check-error": "^1.0.2" } }, - "chainsaw": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", - "integrity": "sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=", - "requires": { - "traverse": ">=0.3.0 <0.4" - } - }, "chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -24704,30 +24499,6 @@ "integrity": "sha512-g/M9sqy3oHe477Ar4voQxWtaPIFw1jTdKZuomOjhCcBx9nHUNn0pu6NopuFFrTh/TRZIKEj+76vLWFu9BNKk+Q==", "dev": true }, - "duplexer2": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", - "integrity": "sha1-ixLauHjA1p4+eJEFFmKjL8a93ME=", - "requires": { - "readable-stream": "^2.0.2" - }, - "dependencies": { - "readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - } - } - }, "duplexify": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.1.tgz", @@ -26001,27 +25772,6 @@ "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", "optional": true }, - "fstream": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz", - "integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==", - "requires": { - "graceful-fs": "^4.1.2", - "inherits": "~2.0.0", - "mkdirp": ">=0.5 0", - "rimraf": "2" - }, - "dependencies": { - "rimraf": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", - "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", - "requires": { - "glob": "^7.1.3" - } - } - } - }, "ftp": { "version": "0.3.10", "resolved": "https://registry.npmjs.org/ftp/-/ftp-0.3.10.tgz", @@ -28122,11 +27872,6 @@ "uc.micro": "^1.0.1" } }, - "listenercount": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/listenercount/-/listenercount-1.0.1.tgz", - "integrity": "sha1-hMinKrWcRyUyFIDJdeZQg0LnCTc=" - }, "load-yaml-file": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/load-yaml-file/-/load-yaml-file-0.2.0.tgz", @@ -31625,11 +31370,6 @@ "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", "devOptional": true }, - "setimmediate": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", - "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=" - }, "setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -32641,11 +32381,6 @@ "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=" }, - "traverse": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", - "integrity": "sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk=" - }, "trim-lines": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/trim-lines/-/trim-lines-3.0.1.tgz", @@ -33028,39 +32763,6 @@ "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" }, - "unzipper": { - "version": "0.10.10", - "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.10.10.tgz", - "integrity": "sha512-wEgtqtrnJ/9zIBsQb8UIxOhAH1eTHfi7D/xvmrUoMEePeI6u24nq1wigazbIFtHt6ANYXdEVTvc8XYNlTurs7A==", - "requires": { - "big-integer": "^1.6.17", - "binary": "~0.3.0", - "bluebird": "~3.4.1", - "buffer-indexof-polyfill": "~1.0.0", - "duplexer2": "~0.1.4", - "fstream": "^1.0.12", - "graceful-fs": "^4.2.2", - "listenercount": "~1.0.1", - "readable-stream": "~2.3.6", - "setimmediate": "~1.0.4" - }, - "dependencies": { - "readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - } - } - }, "update-browserslist-db": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.9.tgz", diff --git a/package.json b/package.json index afaa82e0602..9b7a6a2f771 100644 --- a/package.json +++ b/package.json @@ -149,7 +149,6 @@ "tmp": "^0.2.1", "triple-beam": "^1.3.0", "universal-analytics": "^0.5.3", - "unzipper": "^0.10.10", "update-notifier-cjs": "^5.1.6", "uuid": "^8.3.2", "winston": "^3.0.0", @@ -205,7 +204,6 @@ "@types/tmp": "^0.2.3", "@types/triple-beam": "^1.3.0", "@types/universal-analytics": "^0.4.5", - "@types/unzipper": "^0.10.0", "@types/update-notifier": "^5.1.0", "@types/uuid": "^8.3.1", "@types/ws": "^7.2.3", diff --git a/src/emulator/download.ts b/src/emulator/download.ts index 0841482ce48..bccddaaa141 100644 --- a/src/emulator/download.ts +++ b/src/emulator/download.ts @@ -2,11 +2,11 @@ import * as crypto from "crypto"; import * as fs from "fs-extra"; import * as path from "path"; import * as tmp from "tmp"; -import * as unzipper from "unzipper"; import { EmulatorLogger } from "./emulatorLogger"; import { EmulatorDownloadDetails, DownloadableEmulators } from "./types"; import { FirebaseError } from "../error"; +import { unzip } from "../unzip"; import * as downloadableEmulators from "./downloadableEmulators"; import * as downloadUtils from "../downloadUtils"; @@ -37,10 +37,6 @@ export async function downloadEmulator(name: DownloadableEmulators): Promise setTimeout(f, 2000)); - const executablePath = emulator.binaryPath || emulator.downloadPath; fs.chmodSync(executablePath, 0o755); @@ -78,15 +74,6 @@ export async function downloadExtensionVersion( await new Promise((resolve) => setTimeout(resolve, 1000)); } -function unzip(zipPath: string, unzipDir: string): Promise { - return new Promise((resolve, reject) => { - fs.createReadStream(zipPath) - .pipe(unzipper.Extract({ path: unzipDir })) // eslint-disable-line new-cap - .on("error", reject) - .on("close", resolve); - }); -} - function removeOldFiles( name: DownloadableEmulators, emulator: EmulatorDownloadDetails, diff --git a/src/extensions/extensionsHelper.ts b/src/extensions/extensionsHelper.ts index 4539deb3e2c..11503e446ae 100644 --- a/src/extensions/extensionsHelper.ts +++ b/src/extensions/extensionsHelper.ts @@ -3,11 +3,11 @@ import * as ora from "ora"; import * as semver from "semver"; import * as tmp from "tmp"; import * as fs from "fs-extra"; -import * as unzipper from "unzipper"; import fetch from "node-fetch"; import * as path from "path"; import { marked } from "marked"; +import { createUnzipTransform } from "./../unzip"; const TerminalRenderer = require("marked-terminal"); marked.setOptions({ renderer: new TerminalRenderer(), @@ -676,7 +676,7 @@ export async function publishExtensionVersionFromRemoteRepo(args: { try { const response = await fetch(archiveUri); if (response.ok) { - await response.body.pipe(unzipper.Extract({ path: tempDirectory.name })).promise(); // eslint-disable-line new-cap + await response.body.pipe(createUnzipTransform(tempDirectory.name)).promise(); } } catch (err: any) { throw new FirebaseError( diff --git a/src/test/fixtures/zip-files/node-unzipper-testData/LICENSE b/src/test/fixtures/zip-files/node-unzipper-testData/LICENSE new file mode 100644 index 00000000000..e70827c8553 --- /dev/null +++ b/src/test/fixtures/zip-files/node-unzipper-testData/LICENSE @@ -0,0 +1,25 @@ +Copyright (c) 2012 - 2013 Near Infinity Corporation + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +--- + +Commits in this fork are (c) Ziggy Jonsson (ziggy.jonsson.nyc@gmail.com) +and fall under same licence structure as the original repo (MIT) diff --git a/src/test/fixtures/zip-files/node-unzipper-testData/compressed-cp866/archive.zip b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-cp866/archive.zip new file mode 100644 index 0000000000000000000000000000000000000000..04bd3c9372aa7fea2a053e034aa9ca79a4088687 GIT binary patch literal 163 zcmWIWW@h1H0D*{DWxpVWs_(o&HVAV7@uZ~>AL*4;lw5lH@zc9Uzh5eRdGPXefHxzP xJp(RHDnM-@pa3QjMsPtz7#SoOj`<$(by$3b5uFb3W@Q6uV+6t|AZ-jb0{}7tDPRBq literal 0 HcmV?d00001 diff --git "a/src/test/fixtures/zip-files/node-unzipper-testData/compressed-cp866/inflated/\320\242\320\265\321\201\321\202.txt" "b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-cp866/inflated/\320\242\320\265\321\201\321\202.txt" new file mode 100644 index 00000000000..2f29f70d4d5 --- /dev/null +++ "b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-cp866/inflated/\320\242\320\265\321\201\321\202.txt" @@ -0,0 +1 @@ +Тестовый файл \ No newline at end of file diff --git a/src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/archive.zip b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/archive.zip new file mode 100644 index 0000000000000000000000000000000000000000..e81a6aa7e068c4726c5f589fdd3d87c89f1efb0e GIT binary patch literal 2480 zcmZ{m2|Qcb9>)_2Qd(PMEuQ6}grGD^(ZyD~s%MC_ zx~mksa7?i#@C4ck;1vju!(&M?L@b6#48!?jNVsruHb{nsO+^F+jVH<9YohUw4Ih&^wxTALaJ5DjQFmr-TP4d- z0%DtdtFksniBIF9=Zl-eEz$en#h&;O#FdW<*LpIabakEkp_{EV{$d@8VN0*(dRlek z_(X#v2VT1?LFj20B|^QFGm%BCc=pKMIUBlAvRlk5(g$AJDa;gv^W@%oSla(?b#3!~r$@$PyiX;hj$NagedDqYHdvsT zrCyCQJEWcY9;x;Ut~JdFyf>%SOHgc?BQJ_iZnixxgKrP(XVMPC0&}VbwnW<~zX`@T zYHLh78)l_xtKDwXqE6AuN>by+--yVxlc`cv=bH%;wR42|;7=+|noBxK=MtamuZnjX z>N)whijUDO5y-7Vek4D(Gv$baMD1 zvwtgw1XH%7&P|Zj-aJF<^X?IWgd_tA5dac`BLrZ{$Y2sa zEXmpaPM?kld&T{BtU633y3X%`Vw;Uq%77qjRxajW)_$oiO-%jShM+-BXUSv4ibeee z(-~>WivednjAj;YG`IVEtVv;{S9Jm{9t1v#);Rloldv;M?~>5m6x8JYaDMJb^{*aK%|EKSa< zd&>5PMxx^*lyx7$&kd#2d)ww2JU+cu`bzIY_dod=tO7c;?8>@f%2AlMDYK_D;10|; zOVzD*Ek-J=Bh^@Z^@bI#qvJ=5{;V$==L@RdnX|2{IKr9j)jrQU(CT*)OWue0=^wh^ z%G=IiUaJaE{PFwEFXl&{eUhU)tjD{cGb&}o*<-yI1-PZl z@kVMRA7|eB@oHS&b_03~9XK~RrAJ5eZ%JCNEcYsx*VDHOu{j$016d17=~>?1s0Ljk zJ>DYZ$>RMQnc+%_eBs~%D%j;8ToW?s$4~eIhvW+s1QG-$w4-WR0tp#T47}@Netb%~#|4Bu8`npXGgf}QVFH0aZLY)(Xa?k!K18Q%(hWwWPYjrN zPv0^&dxr~Mrd^xk1uK`n8n5+TEW3@OB@7cduowv|g+$a4lgS93KlaQUX;+*dP-D`n zJ~2IjQc+>|Hc%2-JR_u{2Sa_&I8)XLBkSJu2+u*$s^X@7Dh$}*6Zl?vijJOEi5QEIx_RRdj{f8!?A_;IkukL73@p!vg* zt<()!Xq_P)CiI4x$yzG+*~kc$XiT}^7*(Rgw1{1z&oj#BHR~s8KRoL;%fGSqDbdY< zW56go6npaUD?dmIEDt3bU@aYIlv^D9{O5PDnzwENZ5icj?Sh@@)pjbKmA^jGXe4*C z?)`#0^|aM_NMV8*1Jmg&l;YaaaWl2x$Z^s@WPsKXKDu0p5%bWBd7 z=vB6^Z>$gGy~mK#kpP-1ksMPconyzj(^gLEtk_R}e8fp`aXon>PW#Fa3Lr+$*}kl)0Rh8|HS*T{sbd|ME|6 zh+EZnA!*3>8qdvftIjT036%X_4dQ0G1!9*y&AUe~oNRgdc0R&@s|Q#^`_7L5=s(9p B-P8a8 literal 0 HcmV?d00001 diff --git a/src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/META-INF/container.xml b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/META-INF/container.xml new file mode 100644 index 00000000000..f17cad9aeb0 --- /dev/null +++ b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/META-INF/container.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/content.opf b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/content.opf new file mode 100644 index 00000000000..d3ff63669bb --- /dev/null +++ b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/content.opf @@ -0,0 +1,22 @@ + + + + calibre (3.15.0) [https://calibre-ebook.com] + 2006-03-06T20:06:33+00:00 + 8ee1add8-e31f-4b26-8059-e939a3190706 + en + Author text + Title text + + + + + + + + + + + + + diff --git a/src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/index.html b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/index.html new file mode 100644 index 00000000000..7ddea554403 --- /dev/null +++ b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/index.html @@ -0,0 +1,15 @@ + + + + Blank PDF Document + + + + + + + + +

+ + diff --git a/src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/mimetype b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/mimetype new file mode 100644 index 00000000000..57ef03f24a4 --- /dev/null +++ b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/mimetype @@ -0,0 +1 @@ +application/epub+zip \ No newline at end of file diff --git a/src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/page_styles.css b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/page_styles.css new file mode 100644 index 00000000000..7ee6c2c84a7 --- /dev/null +++ b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/page_styles.css @@ -0,0 +1,4 @@ +@page { + margin-bottom: 5pt; + margin-top: 5pt + } diff --git a/src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/stylesheet.css b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/stylesheet.css new file mode 100644 index 00000000000..749a0321bb9 --- /dev/null +++ b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/stylesheet.css @@ -0,0 +1,11 @@ +.calibre { + display: block; + font-size: 1em; + padding-left: 0; + padding-right: 0; + margin: 0 5pt + } +.calibre1 { + display: block; + margin: 1em 0 + } diff --git a/src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/toc.ncx b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/toc.ncx new file mode 100644 index 00000000000..80db2a6c5f2 --- /dev/null +++ b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-directory-entry/inflated/toc.ncx @@ -0,0 +1,21 @@ + + + + + + + + + + + Title text + + + + + Start + + + + + diff --git a/src/test/fixtures/zip-files/node-unzipper-testData/compressed-flags-set/archive.zip b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-flags-set/archive.zip new file mode 100644 index 0000000000000000000000000000000000000000..015ce233c4600dcf2e099b740615f0a48a68c8fa GIT binary patch literal 1636 zcmWIWW@h1H00Gx14o5HpO0Y14%1JGB6u9D+T-j;?fFk21b@|j0_AcAY}ny z-Aq6hNVlP8+eKy|8-zuny3;aqQa$sEGgDGsGK=&|DoWtSt;00V#E1)Iq6h;E2M5Da zpovv)mrw5m+B5^`L=K>dAk*N6JOvs6!f0-a4U1lP+k)rsd957`hpRP~Nh~kv_ptR#JwzrUH2Q6?=r^TXpz`f^!!JCeT*&J#y3W?4YLXlNNZQ{vZdD^ z?Y_gjV)nJaUiH^=7!CKwFnRB;WO05e$(%PK@ny%(T`V0*3z{B&EM0L=y2|(6wcBso zZ%6%JG<`?mk>vlJ9fvFX{q-K7X|fW1ApGxiJY&hpo_$dh*PUR0kkP=X?{k6w-TbA87aD$E^>4+|DGLuy zO$oTC*48!M%uw{6YVqz`=kxp-OZ_iD;izOeH}9UX&XG$>A;&Ccwj{(@d5D~K6+5$6 zd|`+Bw95%E9v-sh5Z}O1%k@Ee)=Lcr!x;u2{%W>##NXGmnXuM#gJphI`O1&1`%NrF z4Qs<+x*M4MTCTBt@$x%9%U4^j+k81*&Z11YB**2izTPp;yDv^CrF1D6d!1r%|Ehni z;?k_mdNUTvf0Vc)XU<$OOLKXv%l6Z)?y~A?r>94+Gy2N0tIEpLFU~mM;A*QuMfFZ~ z=fiDUAH@3}YW7to}+(gTlF+rXDWQXA_go&P$F2c#w+J>o@G$t%+b4;~q>5IR3z39M=b$ZG!S9Z*kKTuL)?!wn>yTfDF z!t?K57}q80y;GWOEwW^1^OsW2O>9q>@cVwru+BQP-?#KeZpF04sgcUdTplvdn6>C) z@1{p}cNr=!s;qIJCc4pGG5?@;K;xS;M`o{ZKC!ai;Ifk=`%j(TowM95{O(Qn{uDIZ z)5`bRxu7+T*XDdY{z@<0bnffFF(;X4PP9$eJ98)S;E6ZBYZR7$+WBAogOXnF(MIiW z{)vI*m-_{F&0b_3Snv2oE&cM`ny#pmbB`pOnyY#(>pbRm%6U!J6*0wKb!JO8Y`8Mr zwOT2*tw@8JguS4Jm;@}68J0BK0ZEWa wSc+YIMvE{o0n1m+5*OL%2|%L}b06*@S+5i9m literal 0 HcmV?d00001 diff --git a/src/test/fixtures/zip-files/node-unzipper-testData/compressed-flags-set/inflated/dir/fileInsideDir.txt b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-flags-set/inflated/dir/fileInsideDir.txt new file mode 100644 index 00000000000..d81cc0710eb --- /dev/null +++ b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-flags-set/inflated/dir/fileInsideDir.txt @@ -0,0 +1 @@ +42 diff --git a/src/test/fixtures/zip-files/node-unzipper-testData/compressed-flags-set/inflated/file.txt b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-flags-set/inflated/file.txt new file mode 100644 index 00000000000..ac652242866 --- /dev/null +++ b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-flags-set/inflated/file.txt @@ -0,0 +1,11 @@ +node.js rocks + +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Cras commodo molestie nunc, eu pharetra libero accumsan nec. Vestibulum hendrerit, augue ac congue varius, enim metus congue quam, imperdiet gravida diam felis nec dui. Morbi ipsum enim, tristique nec congue a, commodo ac sapien. Praesent semper metus quis diam hendrerit ut condimentum eros lobortis. Aenean faucibus arcu nec leo aliquam tincidunt. Nunc bibendum dictum bibendum. Nunc ultricies pretium lacus, sit amet lobortis quam egestas quis. Fusce viverra magna rhoncus sem posuere non tempus nulla vestibulum. + +Sed aliquet, odio vel condimentum pellentesque, mauris risus iaculis elit, at congue erat mi at ante. In at dictum metus. Ut rutrum mauris felis. Nulla sed risus nunc, eget ultrices est. Nullam gravida diam in arcu vulputate varius. Sed id egestas magna. Ut a libero sapien. + +Integer congue felis ut nisl fringilla ac interdum est pretium. Proin tellus augue, molestie id ultricies placerat, ornare a felis. In eu nibh velit. Pellentesque cursus ultricies fermentum. Mauris eget velit tempor nulla bibendum accumsan sit amet a ante. Morbi rutrum tempor varius. Aenean congue leo vitae mi suscipit ac tempor nibh pulvinar. Maecenas risus eros, sodales quis tincidunt non, vulputate eget orci. Maecenas condimentum lectus pretium orci adipiscing interdum. Sed interdum vehicula urna ut scelerisque. + +Phasellus pellentesque tellus in neque auctor pellentesque adipiscing justo consequat. In tincidunt rhoncus mollis. Suspendisse quis est elit, vel semper lorem. Donec cursus, leo ac fermentum luctus, dui dolor pretium nunc, vel congue eros arcu sit amet enim. Nam nibh orci, laoreet id volutpat eu, aliquet sed ligula. Donec placerat sagittis leo, eget hendrerit nisi varius sed. In pharetra erat non justo interdum id tempus purus tempor. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Suspendisse felis leo, pellentesque tristique iaculis consequat, vestibulum a erat. Curabitur ligula risus, consectetur at adipiscing sit amet, accumsan non justo. Proin ultricies molestie lorem et auctor. Duis commodo varius semper. Ut tempus porttitor dolor nec mattis. Cras massa eros, tincidunt eget placerat a, luctus eu arcu. Nulla ac orci vitae odio dapibus dictum vitae porta erat. + +Duis luctus convallis euismod. Integer orci massa, bibendum eu blandit quis, facilisis lobortis purus. Donec et sapien quis elit fermentum cursus a ut lacus. Nullam tellus felis, congue et pulvinar sit amet, luctus ac augue. Sed massa nunc, dignissim non viverra ac, dictum sit amet erat. Sed nunc tortor, convallis et tristique ut, aliquam ut orci. Integer nec magna vitae elit sagittis accumsan id ac mi. diff --git a/src/test/fixtures/zip-files/node-unzipper-testData/compressed-standard/archive.zip b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-standard/archive.zip new file mode 100644 index 0000000000000000000000000000000000000000..327aab671637f9166c02409b11cb4298f09960e6 GIT binary patch literal 1636 zcmWIWW@h1H00Gx14o5HpO0Y14%1JGB6u9D+T-j;?fFk21b@|j0_AcAY}ny z-Aq6hNVlP8+eKy|8-zuny3;aqQa$sEGgDGsGK=&|DoWtSt;00V#E1)Iq6h;M0|&!W zpovv)mrw5m+B5^`L=K>dAk*N6JOvs6!f0-a4U1lP+k)rsd957`hpRP~Nh~kv_ptR#JwzrUH2Q6?=r^TXpz`f^!!JCeT*&J#y3W?4YLXlNNZQ{vZdD^ z?Y_gjV)nJaUiH^=7!CKwFnRB;WO05e$(%PK@ny%(T`V0*3z{B&EM0L=y2|(6wcBso zZ%6%JG<`?mk>vlJ9fvFX{q-K7X|fW1ApGxiJY&hpo_$dh*PUR0kkP=X?{k6w-TbA87aD$E^>4+|DGLuy zO$oTC*48!M%uw{6YVqz`=kxp-OZ_iD;izOeH}9UX&XG$>A;&Ccwj{(@d5D~K6+5$6 zd|`+Bw95%E9v-sh5Z}O1%k@Ee)=Lcr!x;u2{%W>##NXGmnXuM#gJphI`O1&1`%NrF z4Qs<+x*M4MTCTBt@$x%9%U4^j+k81*&Z11YB**2izTPp;yDv^CrF1D6d!1r%|Ehni z;?k_mdNUTvf0Vc)XU<$OOLKXv%l6Z)?y~A?r>94+Gy2N0tIEpLFU~mM;A*QuMfFZ~ z=fiDUAH@3}YW7to}+(gTlF+rXDWQXA_go&P$F2c#w+J>o@G$t%+b4;~q>5IR3z39M=b$ZG!S9Z*kKTuL)?!wn>yTfDF z!t?K57}q80y;GWOEwW^1^OsW2O>9q>@cVwru+BQP-?#KeZpF04sgcUdTplvdn6>C) z@1{p}cNr=!s;qIJCc4pGG5?@;K;xS;M`o{ZKC!ai;Ifk=`%j(TowM95{O(Qn{uDIZ z)5`bRxu7+T*XDdY{z@<0bnffFF(;X4PP9$eJ98)S;E6ZBYZR7$+WBAogOXnF(MIiW z{)vI*m-_{F&0b_3Snv2oE&cM`ny#pmbB`pOnyY#(>pbRm%6U!J6*0wKb!JO8Y`8Mr zwOT2*tw@8JguS4Jm;@}68J0BK0ZEWa rSc+YIM&l}Rk&T`JG#XL<0!_nG_6B&fvH_I=YXD}3&p4%1JGB6u9D+T-j;?fFk21b@|j0_AcAY}ny z-Aq6hNVlP8+eKy|8-zuny3;aqQa$sEGgDGsGK=&|DoWtSt;00V#E1)Iq6h;M0|&!W zpovv)mrw5m+B5^`L=K>dAk*N6JOvs6!f0-a4U1lP+k)rsd957`hpRP~Nh~kv_ptR#JwzrUH2Q6?=r^TXpz`f^!!JCeT*&J#y3W?4YLXlNNZQ{vZdD^ z?Y_gjV)nJaUiH^=7!CKwFnRB;WO05e$(%PK@ny%(T`V0*3z{B&EM0L=y2|(6wcBso zZ%6%JG<`?mk>vlJ9fvFX{q-K7X|fW1ApGxiJY&hpo_$dh*PUR0kkP=X?{k6w-TbA87aD$E^>4+|DGLuy zO$oTC*48!M%uw{6YVqz`=kxp-OZ_iD;izOeH}9UX&XG$>A;&Ccwj{(@d5D~K6+5$6 zd|`+Bw95%E9v-sh5Z}O1%k@Ee)=Lcr!x;u2{%W>##NXGmnXuM#gJphI`O1&1`%NrF z4Qs<+x*M4MTCTBt@$x%9%U4^j+k81*&Z11YB**2izTPp;yDv^CrF1D6d!1r%|Ehni z;?k_mdNUTvf0Vc)XU<$OOLKXv%l6Z)?y~A?r>94+Gy2N0tIEpLFU~mM;A*QuMfFZ~ z=fiDUAH@3}YW7to}+(gTlF+rXDWQXA_go&P$F2c#w+J>o@G$t%+b4;~q>5IR3z39M=b$ZG!S9Z*kKTuL)?!wn>yTfDF z!t?K57}q80y;GWOEwW^1^OsW2O>9q>@cVwru+BQP-?#KeZpF04sgcUdTplvdn6>C) k@1{p}cNr=!s;qIJCc4pGG5?@;K;xS;M`o{ZKC!YM0P?}-)c^nh literal 0 HcmV?d00001 diff --git a/src/test/fixtures/zip-files/node-unzipper-testData/compressed-standard/inflated/dir/fileInsideDir.txt b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-standard/inflated/dir/fileInsideDir.txt new file mode 100644 index 00000000000..d81cc0710eb --- /dev/null +++ b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-standard/inflated/dir/fileInsideDir.txt @@ -0,0 +1 @@ +42 diff --git a/src/test/fixtures/zip-files/node-unzipper-testData/compressed-standard/inflated/file.txt b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-standard/inflated/file.txt new file mode 100644 index 00000000000..ac652242866 --- /dev/null +++ b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-standard/inflated/file.txt @@ -0,0 +1,11 @@ +node.js rocks + +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Cras commodo molestie nunc, eu pharetra libero accumsan nec. Vestibulum hendrerit, augue ac congue varius, enim metus congue quam, imperdiet gravida diam felis nec dui. Morbi ipsum enim, tristique nec congue a, commodo ac sapien. Praesent semper metus quis diam hendrerit ut condimentum eros lobortis. Aenean faucibus arcu nec leo aliquam tincidunt. Nunc bibendum dictum bibendum. Nunc ultricies pretium lacus, sit amet lobortis quam egestas quis. Fusce viverra magna rhoncus sem posuere non tempus nulla vestibulum. + +Sed aliquet, odio vel condimentum pellentesque, mauris risus iaculis elit, at congue erat mi at ante. In at dictum metus. Ut rutrum mauris felis. Nulla sed risus nunc, eget ultrices est. Nullam gravida diam in arcu vulputate varius. Sed id egestas magna. Ut a libero sapien. + +Integer congue felis ut nisl fringilla ac interdum est pretium. Proin tellus augue, molestie id ultricies placerat, ornare a felis. In eu nibh velit. Pellentesque cursus ultricies fermentum. Mauris eget velit tempor nulla bibendum accumsan sit amet a ante. Morbi rutrum tempor varius. Aenean congue leo vitae mi suscipit ac tempor nibh pulvinar. Maecenas risus eros, sodales quis tincidunt non, vulputate eget orci. Maecenas condimentum lectus pretium orci adipiscing interdum. Sed interdum vehicula urna ut scelerisque. + +Phasellus pellentesque tellus in neque auctor pellentesque adipiscing justo consequat. In tincidunt rhoncus mollis. Suspendisse quis est elit, vel semper lorem. Donec cursus, leo ac fermentum luctus, dui dolor pretium nunc, vel congue eros arcu sit amet enim. Nam nibh orci, laoreet id volutpat eu, aliquet sed ligula. Donec placerat sagittis leo, eget hendrerit nisi varius sed. In pharetra erat non justo interdum id tempus purus tempor. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Suspendisse felis leo, pellentesque tristique iaculis consequat, vestibulum a erat. Curabitur ligula risus, consectetur at adipiscing sit amet, accumsan non justo. Proin ultricies molestie lorem et auctor. Duis commodo varius semper. Ut tempus porttitor dolor nec mattis. Cras massa eros, tincidunt eget placerat a, luctus eu arcu. Nulla ac orci vitae odio dapibus dictum vitae porta erat. + +Duis luctus convallis euismod. Integer orci massa, bibendum eu blandit quis, facilisis lobortis purus. Donec et sapien quis elit fermentum cursus a ut lacus. Nullam tellus felis, congue et pulvinar sit amet, luctus ac augue. Sed massa nunc, dignissim non viverra ac, dictum sit amet erat. Sed nunc tortor, convallis et tristique ut, aliquam ut orci. Integer nec magna vitae elit sagittis accumsan id ac mi. diff --git a/src/test/fixtures/zip-files/node-unzipper-testData/uncompressed/archive.zip b/src/test/fixtures/zip-files/node-unzipper-testData/uncompressed/archive.zip new file mode 100644 index 0000000000000000000000000000000000000000..2d3626d6b6ff0c66cc7d57625270cd85bdb96293 GIT binary patch literal 489 zcmWIWW@h1H00Gx14o5HpO0Y14%1JGB6u9D+Sa7acKoN10%~fMg|5Jkg@<2 z-G-WN7ny;gAS?>iotBxC>X}!ZnUd;~S)^A|Q35v(WF(q#CPrKc6TN}%dB4<0gb!#M z2y*~U1epdm#1?1(2%{O2m!FcVmsPA#l%Jek3^b3CNsbwp4<&%+f`9E6; J=?5SV0|5AfQn~;D literal 0 HcmV?d00001 diff --git a/src/test/fixtures/zip-files/node-unzipper-testData/uncompressed/inflated/dir/fileInsideDir.txt b/src/test/fixtures/zip-files/node-unzipper-testData/uncompressed/inflated/dir/fileInsideDir.txt new file mode 100644 index 00000000000..d81cc0710eb --- /dev/null +++ b/src/test/fixtures/zip-files/node-unzipper-testData/uncompressed/inflated/dir/fileInsideDir.txt @@ -0,0 +1 @@ +42 diff --git a/src/test/fixtures/zip-files/node-unzipper-testData/uncompressed/inflated/file.txt b/src/test/fixtures/zip-files/node-unzipper-testData/uncompressed/inflated/file.txt new file mode 100644 index 00000000000..210e1e1b832 --- /dev/null +++ b/src/test/fixtures/zip-files/node-unzipper-testData/uncompressed/inflated/file.txt @@ -0,0 +1 @@ +node.js rocks diff --git a/src/test/fixtures/zip-files/node-unzipper-testData/zip-slip/archive.zip b/src/test/fixtures/zip-files/node-unzipper-testData/zip-slip/archive.zip new file mode 100644 index 0000000000000000000000000000000000000000..38b3f499de0163e62ca15ce18350a9d9a477a51b GIT binary patch literal 545 zcmWIWW@h1H0D=Au{XYEp{-1?`Y!K#PkYPyA&ri`SsVE5z;bdU8U359h4v0%DxEUB( zzA-W|u!sQFm1JZVD*#cV0!Xz&eqJh90MJm76a&LlprHwl)s`S02)6*So}T`Ippx7I z{nWC|9FT|Lj?Pm62|-=W$Rx*%D=;L0E@xl>dYWNLBZ!3v8dgZqpan~SHzSh>Gwx6T jnE?Vz8bg8PfCLE8QsgiR@MdKLxrhk}K_2A>d6oeH^pk5C literal 0 HcmV?d00001 diff --git a/src/test/fixtures/zip-files/node-unzipper-testData/zip-slip/inflated/good.txt b/src/test/fixtures/zip-files/node-unzipper-testData/zip-slip/inflated/good.txt new file mode 100644 index 00000000000..717599845fd --- /dev/null +++ b/src/test/fixtures/zip-files/node-unzipper-testData/zip-slip/inflated/good.txt @@ -0,0 +1 @@ +this is a good one diff --git a/src/test/fixtures/zip-files/node-unzipper-testData/zip64/archive.zip b/src/test/fixtures/zip-files/node-unzipper-testData/zip64/archive.zip new file mode 100644 index 0000000000000000000000000000000000000000..a2ee1fa33dca48e1ec8dfc7507640bfa09bddeb6 GIT binary patch literal 242 zcmWIWW@Zs#U|`^2Feu@2tb6`HQw7KaVKyKRa&>g^b>%*JLMKe)obQ>FfgY#Nc!n~3 zGWsmC$cFihkI1D@-9^IQUv@AAcr!BTGV7w4^dAb?7(g~az>-D~4KbIIK>%zMNCadf u2n2YuvFSjV47xxF1B_4xjP`)?VKh)5J4k2(lDYtIR*)wcVD13X3=9B3ZZ^#T literal 0 HcmV?d00001 diff --git a/src/test/fixtures/zip-files/node-unzipper-testData/zip64/inflated/README b/src/test/fixtures/zip-files/node-unzipper-testData/zip64/inflated/README new file mode 100644 index 00000000000..ba4fa995f7f --- /dev/null +++ b/src/test/fixtures/zip-files/node-unzipper-testData/zip64/inflated/README @@ -0,0 +1 @@ +This small file is in ZIP64 format. diff --git a/src/test/unzip.spec.ts b/src/test/unzip.spec.ts new file mode 100644 index 00000000000..c11df22182b --- /dev/null +++ b/src/test/unzip.spec.ts @@ -0,0 +1,161 @@ +import { expect } from "chai"; +import * as fs from "fs"; +import * as path from "path"; +import { unzip } from "../unzip"; +import { DownloadDetails } from "../emulator/downloadableEmulators"; +import { Client } from "../apiv2"; +const fixturesDir = path.resolve(__dirname, "./fixtures"); + +const ZIP_FIXTURES_PATH = path.join(fixturesDir, "zip-files"); +const ZIP_TEMPORARY_PATH = path.join(ZIP_FIXTURES_PATH, "temp"); +const ZIP_UNZIPPER_PATH = path.join(ZIP_FIXTURES_PATH, "node-unzipper-testData"); + +describe("unzip", () => { + let uiZipPath: string; + let pubsubZipPath: string; + before(function (done) { + // eslint-disable-next-line @typescript-eslint/no-invalid-this + this.timeout(5000); + (async () => { + const [uiVersion, uiRemoteUrl] = [ + DownloadDetails.ui.version, + DownloadDetails.ui.opts.remoteUrl, + ]; + const [pubsubVersion, pubsubRemoteUrl] = [ + DownloadDetails.pubsub.version, + DownloadDetails.pubsub.opts.remoteUrl, + ]; + + uiZipPath = path.join(ZIP_TEMPORARY_PATH, `ui-v${uiVersion}.zip`); + pubsubZipPath = path.join(ZIP_TEMPORARY_PATH, `pubsub-emulator-v${pubsubVersion}.zip`); + + await fs.promises.mkdir(ZIP_TEMPORARY_PATH, { recursive: true }); + if (!(await fs.promises.access(uiZipPath).catch(() => false))) { + await downloadFile(uiRemoteUrl, uiZipPath); + } + + if (!(await fs.promises.access(pubsubZipPath).catch(() => false))) { + await downloadFile(pubsubRemoteUrl, pubsubZipPath); + } + })() + .then(done) + .catch(done); + }); + + after(async () => { + // await fs.promises.rmdir(ZIP_TEMPORARY_PATH, { recursive: true }); + }); + + it("should unzip a ui emulator zip file", async () => { + await unzip(uiZipPath, path.join(ZIP_TEMPORARY_PATH, "ui")); + + const files = await fs.promises.readdir(ZIP_TEMPORARY_PATH); + expect(files).to.include("ui"); + + const uiFiles = await fs.promises.readdir(path.join(ZIP_TEMPORARY_PATH, "ui")); + expect(uiFiles).to.include("client"); + expect(uiFiles).to.include("server"); + + const serverFiles = await fs.promises.readdir(path.join(ZIP_TEMPORARY_PATH, "ui", "server")); + expect(serverFiles).to.include("server.js"); + }); + + it("should unzip a pubsub emulator zip file", async () => { + await unzip(pubsubZipPath, path.join(ZIP_TEMPORARY_PATH, "pubsub")); + + const files = await fs.promises.readdir(ZIP_TEMPORARY_PATH); + expect(files).to.include("pubsub"); + + const pubsubFiles = await fs.promises.readdir(path.join(ZIP_TEMPORARY_PATH, "pubsub")); + expect(pubsubFiles).to.include("pubsub-emulator"); + + const pubsubEmulatorFiles = await fs.promises.readdir( + path.join(ZIP_TEMPORARY_PATH, "pubsub", "pubsub-emulator") + ); + expect(pubsubEmulatorFiles).to.include("bin"); + expect(pubsubEmulatorFiles).to.include("lib"); + + const binFiles = await fs.promises.readdir( + path.join(ZIP_TEMPORARY_PATH, "pubsub", "pubsub-emulator", "bin") + ); + expect(binFiles).to.include("cloud-pubsub-emulator"); + }); + + const cases = [ + { name: "compressed-cp866" }, + { name: "compressed-directory-entry" }, + { name: "compressed-flags-set" }, + { name: "compressed-standard" }, + { name: "uncompressed" }, + { name: "zip-slip" }, + { name: "zip64" }, + ]; + + for (const { name } of cases) { + it(`should unzip a zip file with ${name} case`, async () => { + const zipPath = path.join(ZIP_UNZIPPER_PATH, name, "archive.zip"); + const unzipPath = path.join(ZIP_TEMPORARY_PATH, name); + await unzip(zipPath, unzipPath); + + // contains a folder "inflated" + const inflatedPath = path.join(ZIP_UNZIPPER_PATH, name, "inflated"); + expect(await fs.promises.stat(inflatedPath)).to.be.ok; + + // // inflated folder's size is "size" + expect(await calculateFolderSize(inflatedPath)).to.eql(await calculateFolderSize(unzipPath)); + }); + } +}); + +async function downloadFile(url: string, targetPath: string): Promise { + const u = new URL(url); + const c = new Client({ urlPrefix: u.origin, auth: false }); + + const writeStream = fs.createWriteStream(targetPath); + + const res = await c.request({ + method: "GET", + path: u.pathname, + queryParams: u.searchParams, + responseType: "stream", + resolveOnHTTPError: true, + }); + + if (res.status !== 200) { + throw new Error( + `Download failed, file "${url}" does not exist. status ${ + res.status + }: ${await res.response.text()}`, + { + cause: new Error( + `Object DownloadDetails from src/emulator/downloadableEmulators.ts contains invalid URL: ${url}` + ), + } + ); + } + + return new Promise((resolve, reject) => { + writeStream.on("finish", () => { + resolve(targetPath); + }); + writeStream.on("error", (err) => { + reject(err); + }); + res.body.pipe(writeStream); + }); +} + +async function calculateFolderSize(folderPath: string): Promise { + const files = await fs.promises.readdir(folderPath); + let size = 0; + for (const file of files) { + const filePath = path.join(folderPath, file); + const stat = await fs.promises.stat(filePath); + if (stat.isDirectory()) { + size += await calculateFolderSize(filePath); + } else { + size += stat.size; + } + } + return size; +} diff --git a/src/unzip.ts b/src/unzip.ts new file mode 100644 index 00000000000..94ae238591a --- /dev/null +++ b/src/unzip.ts @@ -0,0 +1,121 @@ +import * as fs from "fs"; +import * as path from "path"; +import * as zlib from "zlib"; +import { Readable, Transform, TransformCallback } from "stream"; +import { promisify } from "util"; +import { FirebaseError } from "./error"; +import { pipeline } from "stream"; + +const pipelineAsync = promisify(pipeline); + +interface ZipEntry { + compressedSize: number; + uncompressedSize: number; + fileNameLength: number; + extraLength: number; + fileName: string; + headerSize: number; + compressedData: Buffer; +} + +const readUInt32LE = (buf: Buffer, offset: number): number => { + return ( + (buf[offset] | (buf[offset + 1] << 8) | (buf[offset + 2] << 16) | (buf[offset + 3] << 24)) >>> 0 + ); +}; + +const extractEntriesFromBuffer = async (data: Buffer, outputDir: string): Promise => { + let position = 0; + + while (position < data.length) { + const entryHeader = data.slice(position, position + 30); + const entry: ZipEntry = {} as ZipEntry; + + if (readUInt32LE(entryHeader, 0) !== 0x04034b50) { + break; + } + + entry.compressedSize = readUInt32LE(entryHeader, 18); + entry.uncompressedSize = readUInt32LE(entryHeader, 22); + entry.fileNameLength = entryHeader.readUInt16LE(26); + entry.extraLength = entryHeader.readUInt16LE(28); + entry.fileName = data.toString("utf-8", position + 30, position + 30 + entry.fileNameLength); + entry.headerSize = 30 + entry.fileNameLength + entry.extraLength; + entry.compressedData = data.slice( + position + entry.headerSize, + position + entry.headerSize + entry.compressedSize + ); + + const outputFilePath = path.join(outputDir, entry.fileName); + + if (entry.fileName.endsWith("/")) { + await fs.promises.mkdir(outputFilePath, { recursive: true }); + } else { + const parentDir = outputFilePath.substring(0, outputFilePath.lastIndexOf("/")); + await fs.promises.mkdir(parentDir, { recursive: true }); + + const compressionMethod = entryHeader.readUInt16LE(8); + if (compressionMethod === 0) { + // Store (no compression) + await fs.promises.writeFile(outputFilePath, entry.compressedData); + } else if (compressionMethod === 8) { + // Deflate + await pipelineAsync( + Readable.from(entry.compressedData), + zlib.createInflateRaw(), + fs.createWriteStream(outputFilePath) + ); + } else { + throw new FirebaseError(`Unsupported compression method: ${compressionMethod}`); + } + } + + position += entry.headerSize + entry.compressedSize; + } +}; + +export const unzip = async (inputPath: string, outputDir: string): Promise => { + const data = await fs.promises.readFile(inputPath); + await extractEntriesFromBuffer(data, outputDir); +}; + +class UnzipTransform extends Transform { + private chunks: Buffer[] = []; + private _resolve?: () => unknown; + private _reject?: (e: Error) => unknown; + + constructor(private outputDir: string) { + super(); + } + + _transform(chunk: Buffer, _: unknown, callback: TransformCallback): void { + this.chunks.push(chunk); + callback(); + } + + async _flush(callback: TransformCallback): Promise { + try { + await extractEntriesFromBuffer(Buffer.concat(this.chunks), this.outputDir); + callback(); + this._resolve?.(); + } catch (error) { + const firebaseError = new FirebaseError("Unable to unzip the target", { + children: [error], + original: error instanceof Error ? error : undefined, + }); + callback(firebaseError); + this._reject?.(firebaseError); + } + } + + async promise(): Promise { + return new Promise((resolve, reject) => { + this._resolve = resolve; + this._reject = reject; + }); + } +} + +export const createUnzipTransform = (outputDir: string): UnzipTransform => { + return new UnzipTransform(outputDir); +}; From a8381c6f86fb3a14bf2c410911d7582c9d3e800e Mon Sep 17 00:00:00 2001 From: Juraj Carnogursky Date: Mon, 24 Apr 2023 20:26:02 +0200 Subject: [PATCH 02/12] remove unused broken.zip fixture --- .../compressed-standard/broken.zip | Bin 1100 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/test/fixtures/zip-files/node-unzipper-testData/compressed-standard/broken.zip diff --git a/src/test/fixtures/zip-files/node-unzipper-testData/compressed-standard/broken.zip b/src/test/fixtures/zip-files/node-unzipper-testData/compressed-standard/broken.zip deleted file mode 100644 index d13ba6b7ff5c56fb89b1ea6f8acef1d8d81426fa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1100 zcmWIWW@h1H00Gx14o5HpO0Y14%1JGB6u9D+T-j;?fFk21b@|j0_AcAY}ny z-Aq6hNVlP8+eKy|8-zuny3;aqQa$sEGgDGsGK=&|DoWtSt;00V#E1)Iq6h;M0|&!W zpovv)mrw5m+B5^`L=K>dAk*N6JOvs6!f0-a4U1lP+k)rsd957`hpRP~Nh~kv_ptR#JwzrUH2Q6?=r^TXpz`f^!!JCeT*&J#y3W?4YLXlNNZQ{vZdD^ z?Y_gjV)nJaUiH^=7!CKwFnRB;WO05e$(%PK@ny%(T`V0*3z{B&EM0L=y2|(6wcBso zZ%6%JG<`?mk>vlJ9fvFX{q-K7X|fW1ApGxiJY&hpo_$dh*PUR0kkP=X?{k6w-TbA87aD$E^>4+|DGLuy zO$oTC*48!M%uw{6YVqz`=kxp-OZ_iD;izOeH}9UX&XG$>A;&Ccwj{(@d5D~K6+5$6 zd|`+Bw95%E9v-sh5Z}O1%k@Ee)=Lcr!x;u2{%W>##NXGmnXuM#gJphI`O1&1`%NrF z4Qs<+x*M4MTCTBt@$x%9%U4^j+k81*&Z11YB**2izTPp;yDv^CrF1D6d!1r%|Ehni z;?k_mdNUTvf0Vc)XU<$OOLKXv%l6Z)?y~A?r>94+Gy2N0tIEpLFU~mM;A*QuMfFZ~ z=fiDUAH@3}YW7to}+(gTlF+rXDWQXA_go&P$F2c#w+J>o@G$t%+b4;~q>5IR3z39M=b$ZG!S9Z*kKTuL)?!wn>yTfDF z!t?K57}q80y;GWOEwW^1^OsW2O>9q>@cVwru+BQP-?#KeZpF04sgcUdTplvdn6>C) k@1{p}cNr=!s;qIJCc4pGG5?@;K;xS;M`o{ZKC!YM0P?}-)c^nh From d13525931aafbb776b625491d79e48f9e7521a5c Mon Sep 17 00:00:00 2001 From: Juraj Carnogursky Date: Mon, 24 Apr 2023 20:39:21 +0200 Subject: [PATCH 03/12] add changelog record --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04a53eac612..acd7b5790a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ - Added more helpful error messages for the Firebase Hosting GitHub Action (#5749) - Upgrade Firestore emulator to 1.17.4 - Web Frameworks will no longer try to deploy unsupported versions of NodeJS to Cloud Functions (#5733) +- Fix bugs with UI emulator and PubSub emulator not starting correctly (#5714) + - Should resolve #5614 #5677 firebase/firebase-tools-ui#939 firebase/firebase-tools-ui#940 firebase/firebase-tools-ui#942 3de641523 (add changelog record) From 08a7dea2279d6131e4d7b02ed2e4b65010fadccf Mon Sep 17 00:00:00 2001 From: Juraj Carnogursky Date: Mon, 24 Apr 2023 21:17:32 +0200 Subject: [PATCH 04/12] fix timing out test --- src/test/unzip.spec.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/test/unzip.spec.ts b/src/test/unzip.spec.ts index c11df22182b..c2515f1ed3b 100644 --- a/src/test/unzip.spec.ts +++ b/src/test/unzip.spec.ts @@ -46,7 +46,9 @@ describe("unzip", () => { // await fs.promises.rmdir(ZIP_TEMPORARY_PATH, { recursive: true }); }); - it("should unzip a ui emulator zip file", async () => { + it("should unzip a ui emulator zip file", async function () { + // eslint-disable-next-line @typescript-eslint/no-invalid-this + this.timeout(2000); await unzip(uiZipPath, path.join(ZIP_TEMPORARY_PATH, "ui")); const files = await fs.promises.readdir(ZIP_TEMPORARY_PATH); @@ -60,7 +62,9 @@ describe("unzip", () => { expect(serverFiles).to.include("server.js"); }); - it("should unzip a pubsub emulator zip file", async () => { + it("should unzip a pubsub emulator zip file", async function () { + // eslint-disable-next-line @typescript-eslint/no-invalid-this + this.timeout(2000); await unzip(pubsubZipPath, path.join(ZIP_TEMPORARY_PATH, "pubsub")); const files = await fs.promises.readdir(ZIP_TEMPORARY_PATH); From 811de60d8e132ff520cfc752060c41c20f3470cf Mon Sep 17 00:00:00 2001 From: Juraj Carnogursky Date: Tue, 25 Apr 2023 19:33:45 +0200 Subject: [PATCH 05/12] fix use operating system file delimiter --- src/test/unzip.spec.ts | 2 +- src/unzip.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/unzip.spec.ts b/src/test/unzip.spec.ts index c2515f1ed3b..effd6f06957 100644 --- a/src/test/unzip.spec.ts +++ b/src/test/unzip.spec.ts @@ -132,7 +132,7 @@ async function downloadFile(url: string, targetPath: string): Promise { }: ${await res.response.text()}`, { cause: new Error( - `Object DownloadDetails from src/emulator/downloadableEmulators.ts contains invalid URL: ${url}` + `Object DownloadDetails from src${path.sep}emulator${path.sep}downloadableEmulators.ts contains invalid URL: ${url}` ), } ); diff --git a/src/unzip.ts b/src/unzip.ts index 94ae238591a..99811cedf9e 100644 --- a/src/unzip.ts +++ b/src/unzip.ts @@ -48,10 +48,10 @@ const extractEntriesFromBuffer = async (data: Buffer, outputDir: string): Promis const outputFilePath = path.join(outputDir, entry.fileName); - if (entry.fileName.endsWith("/")) { + if (entry.fileName.endsWith(path.sep)) { await fs.promises.mkdir(outputFilePath, { recursive: true }); } else { - const parentDir = outputFilePath.substring(0, outputFilePath.lastIndexOf("/")); + const parentDir = outputFilePath.substring(0, outputFilePath.lastIndexOf(path.sep)); await fs.promises.mkdir(parentDir, { recursive: true }); const compressionMethod = entryHeader.readUInt16LE(8); From 48ad86fe59833e1a7191f7af6af6fb14a5f53931 Mon Sep 17 00:00:00 2001 From: Juraj Carnogursky Date: Tue, 25 Apr 2023 19:58:06 +0200 Subject: [PATCH 06/12] addressing code review comments from @joehan --- CHANGELOG.md | 3 +- scripts/emulator-tests/functions/index.js | 12 ++ scripts/emulator-tests/unzipEmulators.spec.ts | 108 +++++++++++++++++ .../zip-files/node-unzipper-testData/LICENSE | 25 ---- src/test/unzip.spec.ts | 112 +----------------- 5 files changed, 124 insertions(+), 136 deletions(-) create mode 100644 scripts/emulator-tests/functions/index.js create mode 100644 scripts/emulator-tests/unzipEmulators.spec.ts delete mode 100644 src/test/fixtures/zip-files/node-unzipper-testData/LICENSE diff --git a/CHANGELOG.md b/CHANGELOG.md index acd7b5790a0..4057fa6763f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,4 @@ - Added more helpful error messages for the Firebase Hosting GitHub Action (#5749) - Upgrade Firestore emulator to 1.17.4 - Web Frameworks will no longer try to deploy unsupported versions of NodeJS to Cloud Functions (#5733) -- Fix bugs with UI emulator and PubSub emulator not starting correctly (#5714) - - Should resolve #5614 #5677 firebase/firebase-tools-ui#939 firebase/firebase-tools-ui#940 firebase/firebase-tools-ui#942 3de641523 (add changelog record) +- Fixes bug were emulators would not starting correctly due to corrupted ZIP files. (#5614, #5677) diff --git a/scripts/emulator-tests/functions/index.js b/scripts/emulator-tests/functions/index.js new file mode 100644 index 00000000000..09886f1dbb0 --- /dev/null +++ b/scripts/emulator-tests/functions/index.js @@ -0,0 +1,12 @@ +module.exports = (() => { + return { + functionId: require("firebase-functions").https.onRequest((req, resp) => { + return new Promise((resolve) => { + setTimeout(() => { + resp.sendStatus(200); + resolve(); + }, 3000); + }); + }), + }; + })(); diff --git a/scripts/emulator-tests/unzipEmulators.spec.ts b/scripts/emulator-tests/unzipEmulators.spec.ts new file mode 100644 index 00000000000..2840c97cff8 --- /dev/null +++ b/scripts/emulator-tests/unzipEmulators.spec.ts @@ -0,0 +1,108 @@ +import { expect } from "chai"; +import * as fs from "fs"; +import * as path from "path"; +import { unzip } from "../../src/unzip"; +import { DownloadDetails } from "../../src/emulator/downloadableEmulators"; +import { Client } from "../../src/apiv2"; +const fixturesDir = path.resolve(__dirname, "dev"); + +const ZIP_FIXTURES_PATH = path.join(fixturesDir, "zip-files"); +const ZIP_TEMPORARY_PATH = path.join(ZIP_FIXTURES_PATH, "temp"); + +describe("unzipEmulators", () => { + it("should unzip a ui emulator zip file", async () => { + const [uiVersion, uiRemoteUrl] = [ + DownloadDetails.ui.version, + DownloadDetails.ui.opts.remoteUrl, + ]; + + const uiZipPath = path.join(ZIP_TEMPORARY_PATH, `ui-v${uiVersion}.zip`); + + await fs.promises.mkdir(ZIP_TEMPORARY_PATH, { recursive: true }); + if (!(await fs.promises.access(uiZipPath).catch(() => false))) { + await downloadFile(uiRemoteUrl, uiZipPath); + } + + await unzip(uiZipPath, path.join(ZIP_TEMPORARY_PATH, "ui")); + + const files = await fs.promises.readdir(ZIP_TEMPORARY_PATH); + expect(files).to.include("ui"); + + const uiFiles = await fs.promises.readdir(path.join(ZIP_TEMPORARY_PATH, "ui")); + expect(uiFiles).to.include("client"); + expect(uiFiles).to.include("server"); + + const serverFiles = await fs.promises.readdir(path.join(ZIP_TEMPORARY_PATH, "ui", "server")); + expect(serverFiles).to.include("server.js"); + }).timeout(10000); + + it("should unzip a pubsub emulator zip file", async () => { + const [pubsubVersion, pubsubRemoteUrl] = [ + DownloadDetails.pubsub.version, + DownloadDetails.pubsub.opts.remoteUrl, + ]; + + const pubsubZipPath = path.join(ZIP_TEMPORARY_PATH, `pubsub-emulator-v${pubsubVersion}.zip`); + + if (!(await fs.promises.access(pubsubZipPath).catch(() => false))) { + await downloadFile(pubsubRemoteUrl, pubsubZipPath); + } + + await unzip(pubsubZipPath, path.join(ZIP_TEMPORARY_PATH, "pubsub")); + + const files = await fs.promises.readdir(ZIP_TEMPORARY_PATH); + expect(files).to.include("pubsub"); + + const pubsubFiles = await fs.promises.readdir(path.join(ZIP_TEMPORARY_PATH, "pubsub")); + expect(pubsubFiles).to.include("pubsub-emulator"); + + const pubsubEmulatorFiles = await fs.promises.readdir( + path.join(ZIP_TEMPORARY_PATH, "pubsub", "pubsub-emulator") + ); + expect(pubsubEmulatorFiles).to.include("bin"); + expect(pubsubEmulatorFiles).to.include("lib"); + + const binFiles = await fs.promises.readdir( + path.join(ZIP_TEMPORARY_PATH, "pubsub", "pubsub-emulator", "bin") + ); + expect(binFiles).to.include("cloud-pubsub-emulator"); + }).timeout(10000); +}); + +async function downloadFile(url: string, targetPath: string): Promise { + const u = new URL(url); + const c = new Client({ urlPrefix: u.origin, auth: false }); + + const writeStream = fs.createWriteStream(targetPath); + + const res = await c.request({ + method: "GET", + path: u.pathname, + queryParams: u.searchParams, + responseType: "stream", + resolveOnHTTPError: true, + }); + + if (res.status !== 200) { + throw new Error( + `Download failed, file "${url}" does not exist. status ${ + res.status + }: ${await res.response.text()}`, + { + cause: new Error( + `Object DownloadDetails from src${path.sep}emulator${path.sep}downloadableEmulators.ts contains invalid URL: ${url}` + ), + } + ); + } + + return new Promise((resolve, reject) => { + writeStream.on("finish", () => { + resolve(targetPath); + }); + writeStream.on("error", (err) => { + reject(err); + }); + res.body.pipe(writeStream); + }); +} diff --git a/src/test/fixtures/zip-files/node-unzipper-testData/LICENSE b/src/test/fixtures/zip-files/node-unzipper-testData/LICENSE deleted file mode 100644 index e70827c8553..00000000000 --- a/src/test/fixtures/zip-files/node-unzipper-testData/LICENSE +++ /dev/null @@ -1,25 +0,0 @@ -Copyright (c) 2012 - 2013 Near Infinity Corporation - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - ---- - -Commits in this fork are (c) Ziggy Jonsson (ziggy.jonsson.nyc@gmail.com) -and fall under same licence structure as the original repo (MIT) diff --git a/src/test/unzip.spec.ts b/src/test/unzip.spec.ts index effd6f06957..875a952424f 100644 --- a/src/test/unzip.spec.ts +++ b/src/test/unzip.spec.ts @@ -2,8 +2,6 @@ import { expect } from "chai"; import * as fs from "fs"; import * as path from "path"; import { unzip } from "../unzip"; -import { DownloadDetails } from "../emulator/downloadableEmulators"; -import { Client } from "../apiv2"; const fixturesDir = path.resolve(__dirname, "./fixtures"); const ZIP_FIXTURES_PATH = path.join(fixturesDir, "zip-files"); @@ -11,78 +9,12 @@ const ZIP_TEMPORARY_PATH = path.join(ZIP_FIXTURES_PATH, "temp"); const ZIP_UNZIPPER_PATH = path.join(ZIP_FIXTURES_PATH, "node-unzipper-testData"); describe("unzip", () => { - let uiZipPath: string; - let pubsubZipPath: string; - before(function (done) { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(5000); - (async () => { - const [uiVersion, uiRemoteUrl] = [ - DownloadDetails.ui.version, - DownloadDetails.ui.opts.remoteUrl, - ]; - const [pubsubVersion, pubsubRemoteUrl] = [ - DownloadDetails.pubsub.version, - DownloadDetails.pubsub.opts.remoteUrl, - ]; - - uiZipPath = path.join(ZIP_TEMPORARY_PATH, `ui-v${uiVersion}.zip`); - pubsubZipPath = path.join(ZIP_TEMPORARY_PATH, `pubsub-emulator-v${pubsubVersion}.zip`); - - await fs.promises.mkdir(ZIP_TEMPORARY_PATH, { recursive: true }); - if (!(await fs.promises.access(uiZipPath).catch(() => false))) { - await downloadFile(uiRemoteUrl, uiZipPath); - } - - if (!(await fs.promises.access(pubsubZipPath).catch(() => false))) { - await downloadFile(pubsubRemoteUrl, pubsubZipPath); - } - })() - .then(done) - .catch(done); + before(async () => { + await fs.promises.mkdir(ZIP_TEMPORARY_PATH, { recursive: true }); }); after(async () => { - // await fs.promises.rmdir(ZIP_TEMPORARY_PATH, { recursive: true }); - }); - - it("should unzip a ui emulator zip file", async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(2000); - await unzip(uiZipPath, path.join(ZIP_TEMPORARY_PATH, "ui")); - - const files = await fs.promises.readdir(ZIP_TEMPORARY_PATH); - expect(files).to.include("ui"); - - const uiFiles = await fs.promises.readdir(path.join(ZIP_TEMPORARY_PATH, "ui")); - expect(uiFiles).to.include("client"); - expect(uiFiles).to.include("server"); - - const serverFiles = await fs.promises.readdir(path.join(ZIP_TEMPORARY_PATH, "ui", "server")); - expect(serverFiles).to.include("server.js"); - }); - - it("should unzip a pubsub emulator zip file", async function () { - // eslint-disable-next-line @typescript-eslint/no-invalid-this - this.timeout(2000); - await unzip(pubsubZipPath, path.join(ZIP_TEMPORARY_PATH, "pubsub")); - - const files = await fs.promises.readdir(ZIP_TEMPORARY_PATH); - expect(files).to.include("pubsub"); - - const pubsubFiles = await fs.promises.readdir(path.join(ZIP_TEMPORARY_PATH, "pubsub")); - expect(pubsubFiles).to.include("pubsub-emulator"); - - const pubsubEmulatorFiles = await fs.promises.readdir( - path.join(ZIP_TEMPORARY_PATH, "pubsub", "pubsub-emulator") - ); - expect(pubsubEmulatorFiles).to.include("bin"); - expect(pubsubEmulatorFiles).to.include("lib"); - - const binFiles = await fs.promises.readdir( - path.join(ZIP_TEMPORARY_PATH, "pubsub", "pubsub-emulator", "bin") - ); - expect(binFiles).to.include("cloud-pubsub-emulator"); + await fs.promises.rmdir(ZIP_TEMPORARY_PATH, { recursive: true }); }); const cases = [ @@ -111,44 +43,6 @@ describe("unzip", () => { } }); -async function downloadFile(url: string, targetPath: string): Promise { - const u = new URL(url); - const c = new Client({ urlPrefix: u.origin, auth: false }); - - const writeStream = fs.createWriteStream(targetPath); - - const res = await c.request({ - method: "GET", - path: u.pathname, - queryParams: u.searchParams, - responseType: "stream", - resolveOnHTTPError: true, - }); - - if (res.status !== 200) { - throw new Error( - `Download failed, file "${url}" does not exist. status ${ - res.status - }: ${await res.response.text()}`, - { - cause: new Error( - `Object DownloadDetails from src${path.sep}emulator${path.sep}downloadableEmulators.ts contains invalid URL: ${url}` - ), - } - ); - } - - return new Promise((resolve, reject) => { - writeStream.on("finish", () => { - resolve(targetPath); - }); - writeStream.on("error", (err) => { - reject(err); - }); - res.body.pipe(writeStream); - }); -} - async function calculateFolderSize(folderPath: string): Promise { const files = await fs.promises.readdir(folderPath); let size = 0; From dd5adc9310fb5096fbdde5f84ccba8b962293c61 Mon Sep 17 00:00:00 2001 From: Juraj Carnogursky Date: Thu, 27 Apr 2023 16:01:21 +0200 Subject: [PATCH 07/12] better support for Windows path separators TODO: remove console logs after @christhompsongoogle tries it out. --- src/unzip.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/unzip.ts b/src/unzip.ts index 99811cedf9e..1a2c23f200f 100644 --- a/src/unzip.ts +++ b/src/unzip.ts @@ -46,8 +46,11 @@ const extractEntriesFromBuffer = async (data: Buffer, outputDir: string): Promis position + entry.headerSize + entry.compressedSize ); - const outputFilePath = path.join(outputDir, entry.fileName); + entry.fileName = entry.fileName.replace(/\//g, path.sep); + const outputFilePath = path.normalize(path.join(outputDir, entry.fileName)); + + console.log(`Processing entry: \${entry.fileName}`); if (entry.fileName.endsWith(path.sep)) { await fs.promises.mkdir(outputFilePath, { recursive: true }); } else { @@ -57,6 +60,7 @@ const extractEntriesFromBuffer = async (data: Buffer, outputDir: string): Promis const compressionMethod = entryHeader.readUInt16LE(8); if (compressionMethod === 0) { // Store (no compression) + console.log(`Writing file: \${outputFilePath}`); await fs.promises.writeFile(outputFilePath, entry.compressedData); } else if (compressionMethod === 8) { // Deflate From bd5f1858c10ef69fecd3d06d2570822967caffb6 Mon Sep 17 00:00:00 2001 From: Juraj Carnogursky Date: Fri, 28 Apr 2023 12:58:55 +0200 Subject: [PATCH 08/12] addressing code review comments from @joehan --- CHANGELOG.md | 2 +- scripts/emulator-tests/.gitignore | 1 + scripts/emulator-tests/functions/index.js | 12 ------------ src/unzip.ts | 8 ++++++-- 4 files changed, 8 insertions(+), 15 deletions(-) create mode 100644 scripts/emulator-tests/.gitignore delete mode 100644 scripts/emulator-tests/functions/index.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 4057fa6763f..13a8fb3acc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ - Added more helpful error messages for the Firebase Hosting GitHub Action (#5749) - Upgrade Firestore emulator to 1.17.4 - Web Frameworks will no longer try to deploy unsupported versions of NodeJS to Cloud Functions (#5733) -- Fixes bug were emulators would not starting correctly due to corrupted ZIP files. (#5614, #5677) +- Fixes bug where emulators would not start correctly due to corrupted ZIP files. (#5614, #5677) diff --git a/scripts/emulator-tests/.gitignore b/scripts/emulator-tests/.gitignore new file mode 100644 index 00000000000..1fcac224405 --- /dev/null +++ b/scripts/emulator-tests/.gitignore @@ -0,0 +1 @@ +./functions/index.js diff --git a/scripts/emulator-tests/functions/index.js b/scripts/emulator-tests/functions/index.js deleted file mode 100644 index 09886f1dbb0..00000000000 --- a/scripts/emulator-tests/functions/index.js +++ /dev/null @@ -1,12 +0,0 @@ -module.exports = (() => { - return { - functionId: require("firebase-functions").https.onRequest((req, resp) => { - return new Promise((resolve) => { - setTimeout(() => { - resp.sendStatus(200); - resolve(); - }, 3000); - }); - }), - }; - })(); diff --git a/src/unzip.ts b/src/unzip.ts index 1a2c23f200f..15f089b111d 100644 --- a/src/unzip.ts +++ b/src/unzip.ts @@ -5,6 +5,7 @@ import { Readable, Transform, TransformCallback } from "stream"; import { promisify } from "util"; import { FirebaseError } from "./error"; import { pipeline } from "stream"; +import { logger } from "./logger"; const pipelineAsync = promisify(pipeline); @@ -45,12 +46,15 @@ const extractEntriesFromBuffer = async (data: Buffer, outputDir: string): Promis position + entry.headerSize, position + entry.headerSize + entry.compressedSize ); + logger.debug( + `[unzip] Entry: ${entry.fileName} (compressed_size=${entry.compressedSize} bytes, uncompressed_size=${entry.uncompressedSize} bytes)` + ); entry.fileName = entry.fileName.replace(/\//g, path.sep); const outputFilePath = path.normalize(path.join(outputDir, entry.fileName)); - console.log(`Processing entry: \${entry.fileName}`); + logger.debug(`[unzip] Processing entry: ${entry.fileName}`); if (entry.fileName.endsWith(path.sep)) { await fs.promises.mkdir(outputFilePath, { recursive: true }); } else { @@ -60,7 +64,7 @@ const extractEntriesFromBuffer = async (data: Buffer, outputDir: string): Promis const compressionMethod = entryHeader.readUInt16LE(8); if (compressionMethod === 0) { // Store (no compression) - console.log(`Writing file: \${outputFilePath}`); + logger.debug(`[unzip] Writing file: ${outputFilePath}`); await fs.promises.writeFile(outputFilePath, entry.compressedData); } else if (compressionMethod === 8) { // Deflate From 117e7b1d6262399d9ba9921d2d7e4ac1e29ca437 Mon Sep 17 00:00:00 2001 From: Joe Hanley Date: Tue, 2 May 2023 14:31:02 -0700 Subject: [PATCH 09/12] Add support for zip files that use signed data descriptors instead of a full entry header --- src/unzip.ts | 41 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/src/unzip.ts b/src/unzip.ts index 15f089b111d..5430ec4ab83 100644 --- a/src/unzip.ts +++ b/src/unzip.ts @@ -10,6 +10,7 @@ import { logger } from "./logger"; const pipelineAsync = promisify(pipeline); interface ZipEntry { + generalPurposeBitFlag: number; compressedSize: number; uncompressedSize: number; fileNameLength: number; @@ -25,23 +26,49 @@ const readUInt32LE = (buf: Buffer, offset: number): number => { ); }; +const findNextDataDescriptor = async (data:Buffer, offset: number): Promise<[number, number]> => { + const dataDescriptorSignature = 0x08074b50; + let position = offset; + while (position < data.length) { + const potentialDescriptor = data.slice(position, position+16); + if (readUInt32LE(potentialDescriptor, 0) === dataDescriptorSignature) { + logger.debug(`[unzip] found data descriptor sginature @ ${position}`); + const compressedSize = readUInt32LE(potentialDescriptor, 8); + const uncompressedSize = readUInt32LE(potentialDescriptor, 12); + return [compressedSize, uncompressedSize]; + } + position++; + } + throw new FirebaseError("Unable to find compressed and uncompressed size of file in ZIP archive."); +} + const extractEntriesFromBuffer = async (data: Buffer, outputDir: string): Promise => { + // await findHeaders(data); let position = 0; - + logger.debug(`Data is ${data.length}`); while (position < data.length) { const entryHeader = data.slice(position, position + 30); const entry: ZipEntry = {} as ZipEntry; - if (readUInt32LE(entryHeader, 0) !== 0x04034b50) { break; } - + entry.generalPurposeBitFlag = entryHeader.readUint16LE(6); entry.compressedSize = readUInt32LE(entryHeader, 18); entry.uncompressedSize = readUInt32LE(entryHeader, 22); entry.fileNameLength = entryHeader.readUInt16LE(26); entry.extraLength = entryHeader.readUInt16LE(28); entry.fileName = data.toString("utf-8", position + 30, position + 30 + entry.fileNameLength); entry.headerSize = 30 + entry.fileNameLength + entry.extraLength; + let dataDescriptorSize = 0; + if (entry.generalPurposeBitFlag === 8 && entry.compressedSize === 0 && entry.uncompressedSize === 0) { + // If set, entry header won't have compressed or uncompressed size set. + // Need to look ahead to data descriptor to find them. + const [ compressedSize, uncompressedSize ] = await findNextDataDescriptor(data, position); + entry.compressedSize = compressedSize; + entry.uncompressedSize = uncompressedSize; + // If we hit this, we also need to skip over the data descriptor to read the next file + dataDescriptorSize = 16; + } entry.compressedData = data.slice( position + entry.headerSize, position + entry.headerSize + entry.compressedSize @@ -56,18 +83,21 @@ const extractEntriesFromBuffer = async (data: Buffer, outputDir: string): Promis logger.debug(`[unzip] Processing entry: ${entry.fileName}`); if (entry.fileName.endsWith(path.sep)) { + logger.debug(`[unzip] mkdir: ${outputFilePath}`); await fs.promises.mkdir(outputFilePath, { recursive: true }); } else { const parentDir = outputFilePath.substring(0, outputFilePath.lastIndexOf(path.sep)); + logger.debug(`[unzip] else mkdir: ${parentDir}`); await fs.promises.mkdir(parentDir, { recursive: true }); const compressionMethod = entryHeader.readUInt16LE(8); - if (compressionMethod === 0) { + if (entry.compressedSize === 0 || compressionMethod === 0) { // Store (no compression) logger.debug(`[unzip] Writing file: ${outputFilePath}`); await fs.promises.writeFile(outputFilePath, entry.compressedData); } else if (compressionMethod === 8) { // Deflate + logger.debug(`[unzip] deflating: ${outputFilePath}`); await pipelineAsync( Readable.from(entry.compressedData), zlib.createInflateRaw(), @@ -78,12 +108,13 @@ const extractEntriesFromBuffer = async (data: Buffer, outputDir: string): Promis } } - position += entry.headerSize + entry.compressedSize; + position += entry.headerSize + entry.compressedSize + dataDescriptorSize; } }; export const unzip = async (inputPath: string, outputDir: string): Promise => { const data = await fs.promises.readFile(inputPath); + console.log(`data is ${data.byteLength} bytes`); await extractEntriesFromBuffer(data, outputDir); }; From e7bce7dadf374b0c8b1e9537116ec51c6e632349 Mon Sep 17 00:00:00 2001 From: Joe Hanley Date: Tue, 2 May 2023 14:42:12 -0700 Subject: [PATCH 10/12] Formats, fixes uneeded async/await --- src/unzip.ts | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/unzip.ts b/src/unzip.ts index 5430ec4ab83..569924f4839 100644 --- a/src/unzip.ts +++ b/src/unzip.ts @@ -26,24 +26,25 @@ const readUInt32LE = (buf: Buffer, offset: number): number => { ); }; -const findNextDataDescriptor = async (data:Buffer, offset: number): Promise<[number, number]> => { +const findNextDataDescriptor = (data: Buffer, offset: number): [number, number] => { const dataDescriptorSignature = 0x08074b50; let position = offset; while (position < data.length) { - const potentialDescriptor = data.slice(position, position+16); + const potentialDescriptor = data.slice(position, position + 16); if (readUInt32LE(potentialDescriptor, 0) === dataDescriptorSignature) { - logger.debug(`[unzip] found data descriptor sginature @ ${position}`); + logger.debug(`[unzip] found data descriptor signature @ ${position}`); const compressedSize = readUInt32LE(potentialDescriptor, 8); const uncompressedSize = readUInt32LE(potentialDescriptor, 12); return [compressedSize, uncompressedSize]; } position++; } - throw new FirebaseError("Unable to find compressed and uncompressed size of file in ZIP archive."); -} + throw new FirebaseError( + "Unable to find compressed and uncompressed size of file in ZIP archive." + ); +}; const extractEntriesFromBuffer = async (data: Buffer, outputDir: string): Promise => { - // await findHeaders(data); let position = 0; logger.debug(`Data is ${data.length}`); while (position < data.length) { @@ -60,10 +61,14 @@ const extractEntriesFromBuffer = async (data: Buffer, outputDir: string): Promis entry.fileName = data.toString("utf-8", position + 30, position + 30 + entry.fileNameLength); entry.headerSize = 30 + entry.fileNameLength + entry.extraLength; let dataDescriptorSize = 0; - if (entry.generalPurposeBitFlag === 8 && entry.compressedSize === 0 && entry.uncompressedSize === 0) { + if ( + entry.generalPurposeBitFlag === 8 && + entry.compressedSize === 0 && + entry.uncompressedSize === 0 + ) { // If set, entry header won't have compressed or uncompressed size set. // Need to look ahead to data descriptor to find them. - const [ compressedSize, uncompressedSize ] = await findNextDataDescriptor(data, position); + const [compressedSize, uncompressedSize] = findNextDataDescriptor(data, position); entry.compressedSize = compressedSize; entry.uncompressedSize = uncompressedSize; // If we hit this, we also need to skip over the data descriptor to read the next file From 54c88df6b4bf030e680fe920f1f38ad82b1003db Mon Sep 17 00:00:00 2001 From: Joe Hanley Date: Tue, 2 May 2023 16:17:24 -0700 Subject: [PATCH 11/12] Remove unnecessary handling for empty files --- src/unzip.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/unzip.ts b/src/unzip.ts index 569924f4839..72c6af7046f 100644 --- a/src/unzip.ts +++ b/src/unzip.ts @@ -96,7 +96,7 @@ const extractEntriesFromBuffer = async (data: Buffer, outputDir: string): Promis await fs.promises.mkdir(parentDir, { recursive: true }); const compressionMethod = entryHeader.readUInt16LE(8); - if (entry.compressedSize === 0 || compressionMethod === 0) { + if (compressionMethod === 0) { // Store (no compression) logger.debug(`[unzip] Writing file: ${outputFilePath}`); await fs.promises.writeFile(outputFilePath, entry.compressedData); From f052c5c0f54dc96327691c9d52a08da8c574dcba Mon Sep 17 00:00:00 2001 From: Joe Hanley Date: Tue, 2 May 2023 16:19:45 -0700 Subject: [PATCH 12/12] Remove console.log --- src/unzip.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/unzip.ts b/src/unzip.ts index 72c6af7046f..db3c38b014a 100644 --- a/src/unzip.ts +++ b/src/unzip.ts @@ -119,7 +119,6 @@ const extractEntriesFromBuffer = async (data: Buffer, outputDir: string): Promis export const unzip = async (inputPath: string, outputDir: string): Promise => { const data = await fs.promises.readFile(inputPath); - console.log(`data is ${data.byteLength} bytes`); await extractEntriesFromBuffer(data, outputDir); };