test: update proxy setup (#429)

* build: remove --no-clean flag * test: update proxy * test: use a different facebook video url * build: force
microlinkhq · Jun 29, 2021 · 583107d · 583107d
1 parent 8ddcca0
commit 583107d
Show file tree

Hide file tree

Showing 22 changed files with 42 additions and 33 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -25,6 +25,10 @@ jobs:
         run: npm install --no-package-lock
       - name: Test
         run: npm test
+        env:
+          PROXY_USERNAME: ${{ secrets.PROXY_USERNAME }}
+          PROXY_PASSWORD: ${{ secrets.PROXY_PASSWORD }}
+          PROXY_HOST: ${{ secrets.PROXY_HOST }}
       - name: Report
         run: mkdir -p coverage && npx nyc report --reporter=text-lcov > coverage/lcov.info
       - name: Coverage

diff --git a/packages/metascraper-amazon/package.json b/packages/metascraper-amazon/package.json
@@ -34,7 +34,7 @@
     "index.js"
   ],
   "scripts": {
-    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --no-clean --temp-dir ../../.nyc_output mocha test"
+    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --temp-dir ../../.nyc_output mocha test"
   },
   "license": "MIT"
 }
diff --git a/packages/metascraper-audio/package.json b/packages/metascraper-audio/package.json
@@ -34,7 +34,7 @@
     "index.js"
   ],
   "scripts": {
-    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --no-clean --temp-dir ../../.nyc_output mocha test"
+    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --temp-dir ../../.nyc_output mocha test"
   },
   "license": "MIT"
 }
diff --git a/packages/metascraper-clearbit/package.json b/packages/metascraper-clearbit/package.json
@@ -37,7 +37,7 @@
     "index.js"
   ],
   "scripts": {
-    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --no-clean --temp-dir ../../.nyc_output mocha test"
+    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --temp-dir ../../.nyc_output mocha test"
   },
   "license": "MIT",
   "mocha": {

diff --git a/packages/metascraper-description/package.json b/packages/metascraper-description/package.json
@@ -31,7 +31,7 @@
     "index.js"
   ],
   "scripts": {
-    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --no-clean --temp-dir ../../.nyc_output mocha test"
+    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --temp-dir ../../.nyc_output mocha test"
   },
   "license": "MIT"
 }
diff --git a/packages/metascraper-helpers/package.json b/packages/metascraper-helpers/package.json
@@ -53,7 +53,7 @@
     "index.js"
   ],
   "scripts": {
-    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --no-clean --temp-dir ../../.nyc_output mocha test"
+    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --temp-dir ../../.nyc_output mocha test"
   },
   "license": "MIT"
 }
diff --git a/packages/metascraper-iframe/package.json b/packages/metascraper-iframe/package.json
@@ -38,7 +38,7 @@
     "src"
   ],
   "scripts": {
-    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --no-clean --temp-dir ../../.nyc_output mocha test"
+    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --temp-dir ../../.nyc_output mocha test"
   },
   "license": "MIT",
   "mocha": {

diff --git a/packages/metascraper-instagram/package.json b/packages/metascraper-instagram/package.json
@@ -34,7 +34,7 @@
     "index.js"
   ],
   "scripts": {
-    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --no-clean --temp-dir ../../.nyc_output mocha test"
+    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --temp-dir ../../.nyc_output mocha test"
   },
   "license": "MIT",
   "mocha": {

diff --git a/packages/metascraper-lang/package.json b/packages/metascraper-lang/package.json
@@ -31,7 +31,7 @@
     "index.js"
   ],
   "scripts": {
-    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --no-clean --temp-dir ../../.nyc_output mocha test"
+    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --temp-dir ../../.nyc_output mocha test"
   },
   "license": "MIT"
 }
diff --git a/packages/metascraper-logo-favicon/package.json b/packages/metascraper-logo-favicon/package.json
@@ -34,7 +34,7 @@
     "index.js"
   ],
   "scripts": {
-    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --no-clean --temp-dir ../../.nyc_output mocha test"
+    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --temp-dir ../../.nyc_output mocha test"
   },
   "license": "MIT",
   "mocha": {

diff --git a/packages/metascraper-media-provider/package.json b/packages/metascraper-media-provider/package.json
@@ -49,7 +49,7 @@
     "src"
   ],
   "scripts": {
-    "test": "NODE_PATH=.. TZ=UTC DEBUG=metascraper*,-metascraper-media-provider:provider* NODE_ENV=test nyc --silent --no-clean --temp-dir ../../.nyc_output mocha test"
+    "test": "NODE_PATH=.. TZ=UTC DEBUG=metascraper*,-metascraper-media-provider:provider* NODE_ENV=test nyc --silent --temp-dir ../../.nyc_output mocha test"
   },
   "license": "MIT",
   "mocha": {

diff --git a/packages/metascraper-media-provider/src/get-media/provider/generic.js b/packages/metascraper-media-provider/src/get-media/provider/generic.js
@@ -29,7 +29,7 @@ module.exports = ({
   getProxy = constant(false),
   onError = noop,
   timeout = 30000,
-  retry = 5,
+  retry = 2,
   userAgent,
   ...props
 }) => {
@@ -38,7 +38,7 @@ module.exports = ({
     let data = {}
     let isTimeout = false
 
-    const condition = () => !isTimeout && isEmpty(data) && retryCount < retry
+    const condition = () => !isTimeout && isEmpty(data) && retryCount <= retry
 
     const task = async () => {
       await pDoWhilst(async () => {

diff --git a/packages/metascraper-media-provider/test/index.js b/packages/metascraper-media-provider/test/index.js
@@ -2,25 +2,32 @@
 
 const debug = require('debug-logfmt')('metascraper-media-provider:test')
 const { getDomainWithoutSuffix } = require('tldts')
-const parseProxy = require('parse-proxy-uri')
+const parseProxyUri = require('parse-proxy-uri')
 const snapshot = require('snap-shot')
 const should = require('should')
 const isCI = require('is-ci')
 
-const proxy = parseProxy(process.env.PROXY_URI)
+const { PROXY_PASSWORD, PROXY_USERNAME, PROXY_HOST } = process.env
 
-const PROXY_DOMAINS = ['vimeo']
+const proxy =
+  PROXY_PASSWORD && PROXY_USERNAME && PROXY_HOST
+    ? parseProxyUri(
+        `socks5://${PROXY_USERNAME}:${PROXY_PASSWORD}@${PROXY_HOST}`
+      )
+    : undefined
+
+const PROXY_DOMAINS = ['vimeo', 'facebook']
 const PROXY_URLS = ['https://api.twitter.com/1.1/guest/activate.json']
 
-const getProxy = ({ url, retryCount = 1 }) => {
-  if (retryCount === 0) return false
+const getProxy = ({ url, retryCount }) => {
+  if (retryCount !== 1) return false
   if (PROXY_URLS.includes(url)) return proxy
   if (PROXY_DOMAINS.includes(getDomainWithoutSuffix(url))) return proxy
   return false
 }
 
 const metascraper = require('metascraper')([
-  require('..')({ getProxy, timeout: 10000 }),
+  require('..')({ getProxy, timeout: 15000 }),
   require('metascraper-publisher')(),
   require('metascraper-author')(),
   require('metascraper-date')(),
@@ -68,7 +75,7 @@ describe('metascraper-media-provider', () => {
         })
       })
     })
-    ;(isCI ? describe.skip : describe)('vimeo', () => {
+    ;(isCI ? describe.skip : describe.only)('vimeo', () => {
       ;[
         // TODO: uncomment when the issue is resolved
         // https://github.com/ytdl-org/youtube-dl/issues/29205
@@ -121,10 +128,9 @@ describe('metascraper-media-provider', () => {
         should(isUrl(metadata.video)).be.false()
       })
     })
-
-    describe('facebook', () => {
+    ;(isCI ? describe.skip : describe)('facebook', () => {
       ;[
-        'https://www.facebook.com/afcajax/videos/1686831701364171',
+        'https://www.facebook.com/natgeo/videos/10156364216738951/',
         'https://www.facebook.com/cnn/videos/10157803903591509/'
       ].forEach(url => {
         it(url, async () => {
@@ -146,8 +152,7 @@ describe('metascraper-media-provider', () => {
         })
       })
     })
-
-    describe('facebook', () => {
+    ;(isCI ? describe.skip : describe)('facebook', () => {
       ;[
         'https://www.facebook.com/afcajax/videos/1686831701364171',
         'https://www.facebook.com/cnn/videos/10157803903591509/'

diff --git a/packages/metascraper-publisher/package.json b/packages/metascraper-publisher/package.json
@@ -31,7 +31,7 @@
     "index.js"
   ],
   "scripts": {
-    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --no-clean --temp-dir ../../.nyc_output mocha test"
+    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --temp-dir ../../.nyc_output mocha test"
   },
   "license": "MIT"
 }
diff --git a/packages/metascraper-readability/package.json b/packages/metascraper-readability/package.json
@@ -34,7 +34,7 @@
     "index.js"
   ],
   "scripts": {
-    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --no-clean --temp-dir ../../.nyc_output mocha test"
+    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --temp-dir ../../.nyc_output mocha test"
   },
   "license": "MIT",
   "mocha": {

diff --git a/packages/metascraper-soundcloud/package.json b/packages/metascraper-soundcloud/package.json
@@ -33,7 +33,7 @@
     "index.js"
   ],
   "scripts": {
-    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --no-clean --temp-dir ../../.nyc_output mocha test"
+    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --temp-dir ../../.nyc_output mocha test"
   },
   "license": "MIT"
 }
diff --git a/packages/metascraper-spotify/package.json b/packages/metascraper-spotify/package.json
@@ -38,7 +38,7 @@
     "index.js"
   ],
   "scripts": {
-    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --no-clean --temp-dir ../../.nyc_output mocha test"
+    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --temp-dir ../../.nyc_output mocha test"
   },
   "license": "MIT"
 }
diff --git a/packages/metascraper-telegram/package.json b/packages/metascraper-telegram/package.json
@@ -35,7 +35,7 @@
     "index.js"
   ],
   "scripts": {
-    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --no-clean --temp-dir ../../.nyc_output mocha test"
+    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --temp-dir ../../.nyc_output mocha test"
   },
   "license": "MIT",
   "mocha": {

diff --git a/packages/metascraper-uol/package.json b/packages/metascraper-uol/package.json
@@ -36,7 +36,7 @@
     "index.js"
   ],
   "scripts": {
-    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --no-clean --temp-dir ../../.nyc_output mocha test"
+    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --temp-dir ../../.nyc_output mocha test"
   },
   "license": "MIT"
 }
diff --git a/packages/metascraper-video/package.json b/packages/metascraper-video/package.json
@@ -36,7 +36,7 @@
     "index.js"
   ],
   "scripts": {
-    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --no-clean --temp-dir ../../.nyc_output mocha test"
+    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --temp-dir ../../.nyc_output mocha test"
   },
   "license": "MIT"
 }
diff --git a/packages/metascraper-youtube/package.json b/packages/metascraper-youtube/package.json
@@ -37,7 +37,7 @@
     "index.js"
   ],
   "scripts": {
-    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --no-clean --temp-dir ../../.nyc_output mocha test"
+    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --temp-dir ../../.nyc_output mocha test"
   },
   "license": "MIT"
 }
diff --git a/packages/metascraper/package.json b/packages/metascraper/package.json
@@ -75,7 +75,7 @@
     "src"
   ],
   "scripts": {
-    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --no-clean --temp-dir ../../.nyc_output mocha test"
+    "test": "NODE_PATH=.. TZ=UTC NODE_ENV=test nyc --silent --temp-dir ../../.nyc_output mocha test"
   },
   "license": "MIT",
   "mocha": {