From 03e6e28a24e3c9093fd11603b8d751e3c88de237 Mon Sep 17 00:00:00 2001 From: Konrad Dzwinel Date: Tue, 20 Mar 2018 02:19:52 +0100 Subject: [PATCH 1/2] Path robots-parser to work in both node and browser env. --- patches/robots-parser+1.0.2.patch | 36 +++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 patches/robots-parser+1.0.2.patch diff --git a/patches/robots-parser+1.0.2.patch b/patches/robots-parser+1.0.2.patch new file mode 100644 index 000000000000..49aa39ff8c2b --- /dev/null +++ b/patches/robots-parser+1.0.2.patch @@ -0,0 +1,36 @@ +patch-package +--- a/node_modules/robots-parser/Robots.js ++++ b/node_modules/robots-parser/Robots.js +@@ -1,4 +1,4 @@ +-var libUrl = require('url'); ++var URL = (typeof self !== 'undefined' && self.URL) || require('url').URL; + var punycode = require('punycode'); + + /** +@@ -176,7 +176,7 @@ function isPathAllowed(path, rules) { + + + function Robots(url, contents) { +- this._url = libUrl.parse(url); ++ this._url = new URL(url); + this._url.port = this._url.port || 80; + this._url.hostname = punycode.toUnicode(this._url.hostname); + +@@ -262,7 +262,7 @@ Robots.prototype.setPreferredHost = function (url) { + * @return {boolean?} + */ + Robots.prototype.isAllowed = function (url, ua) { +- var parsedUrl = libUrl.parse(url); ++ var parsedUrl = new URL(url); + var userAgent = formatUserAgent(ua || '*'); + + parsedUrl.port = parsedUrl.port || 80; +@@ -277,7 +277,7 @@ Robots.prototype.isAllowed = function (url, ua) { + + var rules = this._rules[userAgent] || this._rules['*'] || []; + +- return isPathAllowed(parsedUrl.path, rules); ++ return isPathAllowed(parsedUrl.pathname + parsedUrl.search, rules); + }; + + /** From 1aba7b0ab97f279e433d62ba75078d071a48bd7c Mon Sep 17 00:00:00 2001 From: Konrad Dzwinel Date: Tue, 20 Mar 2018 02:42:10 +0100 Subject: [PATCH 2/2] Add patch-package that will apply the patch. --- package.json | 5 +- yarn.lock | 154 ++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 157 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 2039ee647b4a..454482327e91 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,8 @@ "plots-smoke": "bash plots/test/smoke.sh", "changelog": "conventional-changelog --config ./build/changelog-generator/index.js --infile changelog.md --same-file", "type-check": "tsc -p .", - "mixed-content": "./lighthouse-cli/index.js --chrome-flags='--headless' --config-path=./lighthouse-core/config/mixed-content.js" + "mixed-content": "./lighthouse-cli/index.js --chrome-flags='--headless' --config-path=./lighthouse-core/config/mixed-content.js", + "prepare": "patch-package" }, "devDependencies": { "@types/chrome": "^0.0.60", @@ -76,6 +77,7 @@ "jsdom": "^9.12.0", "mocha": "^3.2.0", "npm-run-posix-or-windows": "^2.0.2", + "postinstall-prepare": "^1.0.1", "sinon": "^2.3.5", "typescript": "^2.8.0-rc", "zone.js": "^0.7.3" @@ -97,6 +99,7 @@ "mkdirp": "0.5.1", "opn": "4.0.2", "parse-cache-control": "1.0.1", + "patch-package": "^5.1.1", "raven": "^2.2.1", "rimraf": "^2.6.1", "robots-parser": "^1.0.2", diff --git a/yarn.lock b/yarn.lock index 292a06935c3c..3f3d184d1d0d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -238,6 +238,12 @@ ansi-align@^1.1.0: dependencies: string-width "^1.0.1" +ansi-align@^2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/ansi-align/-/ansi-align-2.0.0.tgz#c36aeccba563b89ceb556f3690f0b1d9e3547f7f" + dependencies: + string-width "^2.0.0" + ansi-escapes@^1.1.0: version "1.4.0" resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-1.4.0.tgz#d3a8a83b319aa67793662b13e761c7911422306e" @@ -264,6 +270,12 @@ ansi-styles@^3.1.0: dependencies: color-convert "^1.9.0" +ansi-styles@^3.2.1: + version "3.2.1" + resolved "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" + dependencies: + color-convert "^1.9.0" + archy@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/archy/-/archy-1.0.0.tgz#f9c8c13757cc1dd7bc379ac77b2c62a5c2868c40" @@ -555,6 +567,18 @@ boxen@^1.0.0: term-size "^0.1.0" widest-line "^1.0.0" +boxen@^1.2.1: + version "1.3.0" + resolved "https://registry.npmjs.org/boxen/-/boxen-1.3.0.tgz#55c6c39a8ba58d9c61ad22cd877532deb665a20b" + dependencies: + ansi-align "^2.0.0" + camelcase "^4.0.0" + chalk "^2.0.1" + cli-boxes "^1.0.0" + string-width "^2.0.0" + term-size "^1.2.0" + widest-line "^2.0.0" + brace-expansion@^1.0.0: version "1.1.6" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.6.tgz#7197d7eaa9b87e648390ea61fc66c84427420df9" @@ -675,6 +699,14 @@ chalk@2.1.0, chalk@^2.0.0, chalk@^2.1.0: escape-string-regexp "^1.0.5" supports-color "^4.0.0" +chalk@^2.0.1: + version "2.3.2" + resolved "https://registry.npmjs.org/chalk/-/chalk-2.3.2.tgz#250dc96b07491bfd601e648d66ddf5f60c7a5c65" + dependencies: + ansi-styles "^3.2.1" + escape-string-regexp "^1.0.5" + supports-color "^5.3.0" + chrome-devtools-frontend@1.0.401423: version "1.0.401423" resolved "https://registry.yarnpkg.com/chrome-devtools-frontend/-/chrome-devtools-frontend-1.0.401423.tgz#32a89b8d04e378a494be3c8d63271703be1c04ea" @@ -1061,7 +1093,7 @@ cross-spawn-async@^2.1.1: lru-cache "^4.0.0" which "^1.2.8" -cross-spawn@^5.1.0: +cross-spawn@^5.0.1, cross-spawn@^5.1.0: version "5.1.0" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-5.1.0.tgz#e8bd0efee58fcff6f8f94510a0a554bbfa235449" dependencies: @@ -1413,6 +1445,18 @@ execa@^0.4.0: path-key "^1.0.0" strip-eof "^1.0.0" +execa@^0.7.0: + version "0.7.0" + resolved "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz#944becd34cc41ee32a63a9faf27ad5a65fc59777" + dependencies: + cross-spawn "^5.0.1" + get-stream "^3.0.0" + is-stream "^1.1.0" + npm-run-path "^2.0.0" + p-finally "^1.0.0" + signal-exit "^3.0.0" + strip-eof "^1.0.0" + exit-hook@^1.0.0: version "1.1.1" resolved "https://registry.yarnpkg.com/exit-hook/-/exit-hook-1.1.1.tgz#f05ca233b48c05d54fff07765df8507e95c02ff8" @@ -1650,6 +1694,14 @@ fs-extra@^1.0.0: jsonfile "^2.1.0" klaw "^1.0.0" +fs-extra@^4.0.1: + version "4.0.3" + resolved "https://registry.npmjs.org/fs-extra/-/fs-extra-4.0.3.tgz#0d852122e5bc5beb453fb028e9c0c9bf36340c94" + dependencies: + graceful-fs "^4.1.2" + jsonfile "^4.0.0" + universalify "^0.1.0" + fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" @@ -1830,6 +1882,12 @@ glob@~3.1.21: inherits "1" minimatch "~0.2.11" +global-dirs@^0.1.0: + version "0.1.1" + resolved "https://registry.npmjs.org/global-dirs/-/global-dirs-0.1.1.tgz#b319c0dd4607f353f3be9cca4c72fc148c49f445" + dependencies: + ini "^1.3.4" + global-modules@^0.2.3: version "0.2.3" resolved "https://registry.yarnpkg.com/global-modules/-/global-modules-0.2.3.tgz#ea5a3bed42c6d6ce995a4f8a1269b5dae223828d" @@ -2033,6 +2091,10 @@ has-flag@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-2.0.0.tgz#e8207af1cc7b30d446cc70b734b5e8be18f88d51" +has-flag@^3.0.0: + version "3.0.0" + resolved "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" + has-gulplog@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/has-gulplog/-/has-gulplog-0.1.0.tgz#6414c82913697da51590397dafb12f22967811ce" @@ -2097,6 +2159,10 @@ image-ssim@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/image-ssim/-/image-ssim-0.2.0.tgz#83b42c7a2e6e4b85505477fe6917f5dbc56420e5" +import-lazy@^2.1.0: + version "2.1.0" + resolved "https://registry.npmjs.org/import-lazy/-/import-lazy-2.1.0.tgz#05698e3d45c88e8d7e9d92cb0584e77f096f3e43" + imurmurhash@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" @@ -2239,6 +2305,13 @@ is-glob@^2.0.0, is-glob@^2.0.1: dependencies: is-extglob "^1.0.0" +is-installed-globally@^0.1.0: + version "0.1.0" + resolved "https://registry.npmjs.org/is-installed-globally/-/is-installed-globally-0.1.0.tgz#0dfd98f5a9111716dd535dda6492f67bf3d25a80" + dependencies: + global-dirs "^0.1.0" + is-path-inside "^1.0.0" + is-my-json-valid@^2.12.4: version "2.15.0" resolved "https://registry.yarnpkg.com/is-my-json-valid/-/is-my-json-valid-2.15.0.tgz#936edda3ca3c211fd98f3b2d3e08da43f7b2915b" @@ -2510,6 +2583,12 @@ jsonfile@^2.1.0: optionalDependencies: graceful-fs "^4.1.6" +jsonfile@^4.0.0: + version "4.0.0" + resolved "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz#8771aae0799b64076b76640fca058f9c10e33ecb" + optionalDependencies: + graceful-fs "^4.1.6" + jsonify@~0.0.0: version "0.0.0" resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.0.tgz#2c74b6ee41d93ca51b7b5aaee8f503631d252a73" @@ -3026,6 +3105,12 @@ npm-run-path@^1.0.0: dependencies: path-key "^1.0.0" +npm-run-path@^2.0.0: + version "2.0.2" + resolved "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz#35a9232dfa35d7067b4cb2ddf2357b1871536c5f" + dependencies: + path-key "^2.0.0" + npm-run-posix-or-windows@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/npm-run-posix-or-windows/-/npm-run-posix-or-windows-2.0.2.tgz#74e894702ae34ea338502d04b500c1dec836736e" @@ -3141,6 +3226,10 @@ osenv@^0.1.3: os-homedir "^1.0.0" os-tmpdir "^1.0.0" +p-finally@^1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae" + p-limit@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-1.1.0.tgz#b07ff2d9a5d88bec806035895a2bab66a27988bc" @@ -3201,6 +3290,19 @@ parse5@^1.5.1: version "1.5.1" resolved "https://registry.yarnpkg.com/parse5/-/parse5-1.5.1.tgz#9b7f3b0de32be78dc2401b17573ccaf0f6f59d94" +patch-package@^5.1.1: + version "5.1.1" + resolved "https://registry.npmjs.org/patch-package/-/patch-package-5.1.1.tgz#e5e82fe08bed760b773b8eb73a7bcb7c1634f802" + dependencies: + chalk "^1.1.3" + cross-spawn "^5.1.0" + fs-extra "^4.0.1" + minimist "^1.2.0" + rimraf "^2.6.1" + slash "^1.0.0" + tmp "^0.0.31" + update-notifier "^2.2.0" + path-exists@2.1.0, path-exists@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-2.1.0.tgz#0feb6c64f0fc518d9a754dd5efb62c7022761f4b" @@ -3227,6 +3329,10 @@ path-key@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/path-key/-/path-key-1.0.0.tgz#5d53d578019646c0d68800db4e146e6bdc2ac7af" +path-key@^2.0.0: + version "2.0.1" + resolved "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz#411cadb574c5a140d3a4b1910d40d80cc9f40b40" + path-root-regex@^0.1.0: version "0.1.2" resolved "https://registry.yarnpkg.com/path-root-regex/-/path-root-regex-0.1.2.tgz#bfccdc8df5b12dc52c8b43ec38d18d72c04ba96d" @@ -3279,6 +3385,10 @@ pluralize@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-7.0.0.tgz#298b89df8b93b0221dbf421ad2b1b1ea23fc6777" +postinstall-prepare@^1.0.1: + version "1.0.1" + resolved "https://registry.npmjs.org/postinstall-prepare/-/postinstall-prepare-1.0.1.tgz#dac9b5d91b054389141b13c0192eb68a0aa002b5" + prelude-ls@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54" @@ -3947,6 +4057,12 @@ supports-color@^4.0.0: dependencies: has-flag "^2.0.0" +supports-color@^5.3.0: + version "5.3.0" + resolved "https://registry.npmjs.org/supports-color/-/supports-color-5.3.0.tgz#5b24ac15db80fa927cf5227a4a33fd3c4c7676c0" + dependencies: + has-flag "^3.0.0" + symbol-tree@^3.2.1: version "3.2.2" resolved "https://registry.yarnpkg.com/symbol-tree/-/symbol-tree-3.2.2.tgz#ae27db38f660a7ae2e1c3b7d1bc290819b8519e6" @@ -3982,6 +4098,12 @@ term-size@^0.1.0: dependencies: execa "^0.4.0" +term-size@^1.2.0: + version "1.2.0" + resolved "https://registry.npmjs.org/term-size/-/term-size-1.2.0.tgz#458b83887f288fc56d6fffbfad262e26638efa69" + dependencies: + execa "^0.7.0" + text-encoding@0.6.4: version "0.6.4" resolved "https://registry.yarnpkg.com/text-encoding/-/text-encoding-0.6.4.tgz#e399a982257a276dae428bb92845cb71bdc26d19" @@ -4043,6 +4165,12 @@ tmp@^0.0.29: dependencies: os-tmpdir "~1.0.1" +tmp@^0.0.31: + version "0.0.31" + resolved "https://registry.npmjs.org/tmp/-/tmp-0.0.31.tgz#8f38ab9438e17315e5dbd8b3657e8bfb277ae4a7" + dependencies: + os-tmpdir "~1.0.1" + tmp@^0.0.33: version "0.0.33" resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9" @@ -4132,6 +4260,10 @@ unique-string@^1.0.0: dependencies: crypto-random-string "^1.0.0" +universalify@^0.1.0: + version "0.1.1" + resolved "https://registry.npmjs.org/universalify/-/universalify-0.1.1.tgz#fa71badd4437af4c148841e3b3b165f9e9e590b7" + unzip-response@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/unzip-response/-/unzip-response-2.0.1.tgz#d2f0f737d16b0615e72a6935ed04214572d56f97" @@ -4149,6 +4281,20 @@ update-notifier@^2.1.0: semver-diff "^2.0.0" xdg-basedir "^3.0.0" +update-notifier@^2.2.0: + version "2.3.0" + resolved "https://registry.npmjs.org/update-notifier/-/update-notifier-2.3.0.tgz#4e8827a6bb915140ab093559d7014e3ebb837451" + dependencies: + boxen "^1.2.1" + chalk "^2.0.1" + configstore "^3.0.0" + import-lazy "^2.1.0" + is-installed-globally "^0.1.0" + is-npm "^1.0.0" + latest-version "^3.0.0" + semver-diff "^2.0.0" + xdg-basedir "^3.0.0" + url-parse-lax@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/url-parse-lax/-/url-parse-lax-1.0.0.tgz#7af8f303645e9bd79a272e7a14ac68bc0609da73" @@ -4277,6 +4423,12 @@ widest-line@^1.0.0: dependencies: string-width "^1.0.1" +widest-line@^2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/widest-line/-/widest-line-2.0.0.tgz#0142a4e8a243f8882c0233aa0e0281aa76152273" + dependencies: + string-width "^2.1.1" + window-size@0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.1.0.tgz#5438cd2ea93b202efa3a19fe8887aee7c94f9c9d"