feat(logo): ensure logo from markup is reachable

packages/metascraper-logo-favicon/README.md

@@ -48,9 +48,9 @@ Type: `function`
 It will be used for picking the value to extract from a set of favicon detected on the markup.
 
 ```js
-const pickFn = (sizes, pickDefault) => {
+const pickFn = (sizes, { pickDefault }) => {
   const appleTouchIcon = sizes.find((item) => item.rel.includes('apple'))
-  return appleTouchIcon || pickDefault(sizes)
+  return (appleTouchIcon || pickDefault(sizes)).url
 }
 
 const metascraper = require('metascraper')([

packages/metascraper-logo-favicon/src/index.js

@@ -1,20 +1,13 @@
 'use strict'
 
-const { isEmpty, first, toNumber, chain, get, orderBy } = require('lodash')
+const { logo, parseUrl, normalizeUrl, toRule } = require('@metascraper/helpers')
+const { isEmpty, first, toNumber, chain, orderBy } = require('lodash')
 const reachableUrl = require('reachable-url')
 const memoize = require('@keyvhq/memoize')
 
-const {
-  logo,
-  parseUrl,
-  normalizeUrl,
-  toRule,
-  logo: logoFn
-} = require('@metascraper/helpers')
-
 const SIZE_REGEX_BY_X = /\d+x\d+/
 
-const toLogo = toRule(logoFn)
+const toLogo = toRule(logo)
 
 const toSize = (input, url) => {
   if (isEmpty(input)) return
@@ -58,27 +51,27 @@ const getSize = (url, sizes) =>
   toSize(first(url.match(SIZE_REGEX_BY_X)), url) ||
   toSize.fallback(url)
 
-const getDomNodeSizes = (domNodes, attr) =>
+const getDomNodeSizes = (domNodes, attr, url) =>
   chain(domNodes)
     .reduce((acc, domNode) => {
-      const url = domNode.attribs[attr]
-      if (!url) return acc
+      const relativeUrl = domNode.attribs[attr]
+      if (!relativeUrl) return acc
       return [
         ...acc,
         {
           ...domNode.attribs,
-          url,
+          url: normalizeUrl(url, relativeUrl),
           size: getSize(url, domNode.attribs.sizes)
         }
       ]
     }, [])
     .value()
 
-const getSizes = ($, collection) =>
+const getSizes = ($, collection, url) =>
   chain(collection)
     .reduce((acc, { tag, attr }) => {
       const domNodes = $(tag).get()
-      return [...acc, ...getDomNodeSizes(domNodes, attr)]
+      return [...acc, ...getDomNodeSizes(domNodes, attr, url)]
     }, [])
     .value()
 
@@ -87,7 +80,16 @@ const sizeSelectors = [
   { tag: 'meta[name*="msapplication" i]', attr: 'content' } // Windows 8, Internet Explorer 11 Tiles
 ]
 
-const pickBiggerSize = sizes => {
+const firstReachable = async (domNodeSizes, gotOpts) => {
+  for (const { url } of domNodeSizes) {
+    const response = await reachableUrl(url, gotOpts)
+    if (reachableUrl.isReachable(response)) {
+      return response.url
+    }
+  }
+}
+
+const pickBiggerSize = async (sizes, { gotOpts } = {}) => {
   const sorted = sizes.reduce(
     (acc, item) => {
       acc[item.size.square ? 'square' : 'nonSquare'].push(item)
@@ -97,8 +99,8 @@ const pickBiggerSize = sizes => {
   )
 
   return (
-    first(pickBiggerSize.sortBySize(sorted.square)) ||
-    first(pickBiggerSize.sortBySize(sorted.nonSquare))
+    (await firstReachable(pickBiggerSize.sortBySize(sorted.square), gotOpts)) ||
+    (await firstReachable(pickBiggerSize.sortBySize(sorted.nonSquare), gotOpts))
   )
 }
 
@@ -170,10 +172,9 @@ module.exports = ({
   const rootFavicon = createRootFavicon({ getLogo, withRootFavicon })
   return {
     logo: [
-      toLogo($ => {
-        const sizes = getSizes($, sizeSelectors)
-        const size = pickFn(sizes, pickBiggerSize)
-        return get(size, 'url')
+      toLogo(async ($, url) => {
+        const sizes = getSizes($, sizeSelectors, url)
+        return await pickFn(sizes, { gotOpts, pickBiggerSize })
       }),
       ({ url }) => getLogo(normalizeUrl(url)),
       rootFavicon
@@ -185,3 +186,4 @@ module.exports.favicon = favicon
 module.exports.google = google
 module.exports.createRootFavicon = createRootFavicon
 module.exports.createGetLogo = createGetLogo
+module.exports.pickBiggerSize = pickBiggerSize

packages/metascraper-logo-favicon/test/favicon.js

@@ -4,7 +4,12 @@ const test = require('ava')
 
 const { favicon } = require('..')
 
-test('with { contentType: \'image/vnd.microsoft.icon\' }', async t => {
+test('return undefined if favicon is not reachable', async t => {
+  const url = 'https://idontexist.lol'
+  t.is(await favicon(url), undefined)
+})
+
+test("with { contentType: 'image/vnd.microsoft.icon' }", async t => {
   const url = 'https://microlink.io/'
   t.is(await favicon(url), 'https://microlink.io/favicon.ico')
 })

packages/metascraper-logo-favicon/test/google.js

@@ -5,7 +5,7 @@ const got = require('got')
 
 const { google } = require('..')
 
-test('return undefined under no logo', async t => {
+test('return undefined if favicon is not reachable', async t => {
   const url = 'https://idontexist.lol'
   t.is(await google(url), undefined)
 })

packages/metascraper-logo-favicon/test/index.js

@@ -29,10 +29,12 @@ test('provide `keyvOpts`', async t => {
 test('provide `pickFn`', async t => {
   const url = 'https://www.theverge.com'
   const html = await readFile(resolve(__dirname, 'fixtures/theverge.html'))
-  const pickFn = (sizes, pickDefault) => {
+
+  const pickFn = (sizes, { pickDefault }) => {
     const appleTouchIcon = sizes.find(item => item.rel.includes('apple'))
-    return appleTouchIcon || pickDefault(sizes)
+    return (appleTouchIcon || pickDefault(sizes)).url
   }
+
   const metascraper = createMetascraper({ pickFn })
   const metadata = await metascraper({ url, html })
   t.is(
@@ -60,31 +62,27 @@ test('create an absolute favicon url if the logo is not present', async t => {
 })
 
 test('get the biggest icon possible', async t => {
-  const url = 'https://www.microsoft.com/design/fluent'
+  const url = 'https://cdn.microlink.io'
   const metascraper = createMetascraper()
   const html = createHtml([
-    '<link rel="apple-touch-icon" href="assets/favicons/favicon-57.png">',
-    '<link rel="apple-touch-icon" sizes="114x114" href="assets/favicons/favicon-114.png">',
-    '<link rel="apple-touch-icon" sizes="72x72" href="assets/favicons/favicon-72.png">',
-    '<link rel="apple-touch-icon" sizes="144x144" href="assets/favicons/favicon-144.png">',
-    '<link rel="apple-touch-icon" sizes="60x60" href="assets/favicons/favicon-60.png">',
-    '<link rel="apple-touch-icon" sizes="120x120" href="assets/favicons/favicon-120.png">',
-    '<link rel="apple-touch-icon" sizes="76x76" href="assets/favicons/favicon-76.png">',
-    '<link rel="apple-touch-icon" sizes="152x152" href="assets/favicons/favicon-152.png">',
-    '<link rel="apple-touch-icon" sizes="180x180" href="assets/favicons/favicon-180.png"></link>',
-    '<link rel="shortcut icon" href="assets/favicons/favicon.ico">',
-    '<link rel="icon" sizes="16x16 32x32 64x64" href="assets/favicons/favicon.ico">',
-    '<link rel="icon" type="image/png" sizes="196x196" href="assets/favicons/favicon-192.png">',
-    '<link rel="icon" type="image/png" sizes="160x160" href="assets/favicons/favicon-160.png">',
-    '<link rel="icon" type="image/png" sizes="96x96" href="assets/favicons/favicon-96.png">',
-    '<link rel="icon" type="image/png" sizes="64x64" href="assets/favicons/favicon-64.png">',
-    '<link rel="icon" type="image/png" sizes="32x32" href="assets/favicons/favicon-32.png">',
-    '<link rel="icon" type="image/png" sizes="16x16" href="assets/favicons/favicon-16.png">'
+    '<link rel="apple-touch-icon-precomposed" sizes="57x57" href="/logo/apple-touch-icon-57x57.png">',
+    '<link rel="apple-touch-icon-precomposed" sizes="114x114" href="/logo/apple-touch-icon-114x114.png">',
+    '<link rel="apple-touch-icon-precomposed" sizes="72x72" href="/logo/apple-touch-icon-72x72.png">',
+    '<link rel="apple-touch-icon-precomposed" sizes="144x144" href="/logo/apple-touch-icon-144x144.png">',
+    '<link rel="apple-touch-icon-precomposed" sizes="60x60" href="/logo/apple-touch-icon-60x60.png">',
+    '<link rel="apple-touch-icon-precomposed" sizes="120x120" href="/logo/apple-touch-icon-120x120.png">',
+    '<link rel="apple-touch-icon-precomposed" sizes="76x76" href="/logo/apple-touch-icon-76x76.png">',
+    '<link rel="apple-touch-icon-precomposed" sizes="152x152" href="/logo/apple-touch-icon-152x152.png">',
+    '<link rel="icon" type="image/png" href="/logo/favicon-196x196.png" sizes="196x196">',
+    '<link rel="icon" type="image/png" href="/logo/favicon-96x96.png" sizes="96x96">',
+    '<link rel="icon" type="image/png" href="/logo/favicon-32x32.png" sizes="32x32">',
+    '<link rel="icon" type="image/png" href="/logo/favicon-16x16.png" sizes="16x16">',
+    '<link rel="icon" type="image/png" href="/logo/favicon-128.png" sizes="128x128">'
   ])
   const metadata = await metascraper({ url, html })
   t.is(
     metadata.logo,
-    'https://www.microsoft.com/design/assets/favicons/favicon-192.png'
+    'https://cdn.microlink.io/logo/favicon-196x196.png'
   )
 })
 
@@ -135,35 +133,35 @@ test('detect `rel="icon"`', async t => {
 })
 
 test('detect `rel="apple-touch-icon-precomposed"`', async t => {
-  const url = 'https://github.com'
+  const url = 'https://cdn.microlink.io/'
   const metascraper = createMetascraper()
   const html = createHtml([
-    '<link rel="apple-touch-icon-precomposed" sizes="114x114" href="assets/favicons/favicon-114.png">'
+    '<link rel="apple-touch-icon-precomposed" sizes="144x144" href="logo/apple-touch-icon-144x144.png">'
   ])
   const metadata = await metascraper({ url, html })
-  t.is(metadata.logo, 'https://github.com/assets/favicons/favicon-114.png')
+  t.is(metadata.logo, 'https://cdn.microlink.io/logo/apple-touch-icon-144x144.png')
 })
 
 test('detect `rel="apple-touch-icon"`', async t => {
-  const url = 'https://github.com'
+  const url = 'https://cdn.microlink.io/'
   const metascraper = createMetascraper()
   const html = createHtml([
-    '<link rel="apple-touch-icon" sizes="114x114" href="assets/favicons/favicon-114.png">'
+    '<link rel="apple-touch-icon" sizes="144x144" href="logo/apple-touch-icon-144x144.png">'
   ])
   const metadata = await metascraper({ url, html })
-  t.is(metadata.logo, 'https://github.com/assets/favicons/favicon-114.png')
+  t.is(metadata.logo, 'https://cdn.microlink.io/logo/apple-touch-icon-144x144.png')
 })
 
 test('detect `rel="shortcut icon"`', async t => {
-  const url = 'https://www.microsoft.com/design/fluent'
+  const url = 'https://cdn.microlink.io/'
   const metascraper = createMetascraper()
   const html = createHtml([
-    '<link rel="shortcut icon" href="assets/favicons/favicon.ico">'
+    '<link rel="shortcut icon" href="logo/favicon.ico">'
   ])
   const metadata = await metascraper({ url, html })
   t.is(
     metadata.logo,
-    'https://www.microsoft.com/design/assets/favicons/favicon.ico'
+    'https://cdn.microlink.io/logo/favicon.ico'
   )
 })
 
@@ -186,23 +184,23 @@ test('square logos has priority', async t => {
   const metascraper = createMetascraper()
   const html = createHtml([
     '<meta name="msapplication-wide310x150logo" content="https://s.yimg.com/kw/assets/eng-e-558x270.png">',
-    '<link rel="apple-touch-icon" sizes="114x114" href="assets/favicons/favicon-114.png">'
+    '<link rel="apple-touch-icon" sizes="114x114" href="https://s.yimg.com/kw/assets/apple-touch-icon-152x152.png">'
   ])
   const metadata = await metascraper({ url, html })
   t.is(
     metadata.logo,
-    'https://www.engadget.com/assets/favicons/favicon-114.png'
+    'https://s.yimg.com/kw/assets/apple-touch-icon-152x152.png'
   )
 })
 
 test('detect size from `sizes`', async t => {
-  const url = 'https://example.com'
+  const url = 'https://cdn.microlink.io/'
   const metascraper = createMetascraper()
   const html = createHtml([
-    '<link rel="icon" sizes="16x16 32x32 64x64" href="assets/favicons/favicon.ico">'
+    '<link rel="icon" sizes="16x16 32x32 64x64" href="logo/favicon.ico">'
   ])
   const metadata = await metascraper({ url, html })
-  t.is(metadata.logo, 'https://example.com/assets/favicons/favicon.ico')
+  t.is(metadata.logo, 'https://cdn.microlink.io/logo/favicon.ico')
 })
 
 test('use non square logo as in the worst scenario', async t => {

packages/metascraper-logo-favicon/test/pick-fn.js

@@ -0,0 +1,44 @@
+'use strict'
+
+const test = require('ava')
+
+const { pickBiggerSize } = require('..')
+
+test('ensure logo is reachable', async t => {
+  const sizes = [
+    {
+      rel: 'icon',
+      type: 'image/png',
+      sizes: '16x16',
+      href: 'https://www.android.com/=w16',
+      url: 'https://www.android.com/=w16',
+      size: { height: 16, width: 16, square: true, priority: 80 }
+    },
+    {
+      rel: 'icon',
+      type: 'image/png',
+      sizes: '32x32',
+      href: 'https://www.android.com/=w32',
+      url: 'https://www.android.com/=w32',
+      size: { height: 32, width: 32, square: true, priority: 160 }
+    },
+    {
+      rel: 'apple-touch-icon-precomposed',
+      sizes: '180x180',
+      href: 'https://www.android.com/=w180',
+      url: 'https://www.android.com/=w180',
+      size: { height: 180, width: 180, square: true, priority: 900 }
+    },
+    {
+      rel: 'shortcut icon',
+      href: 'https://www.android.com/static/img/favicon.ico?cache=33c79c9',
+      url: 'https://www.android.com/static/img/favicon.ico?cache=33c79c9',
+      size: { width: 0, height: 0, square: true, priority: 5 }
+    }
+  ]
+
+  t.is(
+    await pickBiggerSize(sizes),
+    'https://www.android.com/static/img/favicon.ico?cache=33c79c9'
+  )
+})