Merge pull request #13115 from brave/feature/publisher-request-api

Add publishers media info request API

app/browser/api/ledger.js

@@ -1408,6 +1408,77 @@ const clientprep = () => {
   _internal.verboseP = ledgerClient.prototype.boolion(process.env.LEDGER_PUBLISHER_VERBOSE)
 }
 
+/**
+ * This callback that we do it in roundtrips
+ *
+ * @callback roundtripCallback
+ * @param {Error|null} error - error when doing a request, null if no errors
+ * @param {object} response - response object that we get from request.request
+ * @param {any} payload - data that we get from the request
+ */
+
+/**
+ * Round trip for fetching data (scrap data from html) inside window process
+ * @param {object} params - contains params from roundtrip
+ * @param {string} params.url - url of the site that we want to scrap
+ * @param {string} params.verboseP - tells us if we want to log the process or not
+ * @param {roundtripCallback} callback - The callback that handles the response
+ */
+const roundTripFromWindow = (params, callback) => {
+  if (!callback) {
+    return
+  }
+
+  if (!params || !params.url) {
+    if (params && params.verboseP) {
+      console.log(`We are missing url. Params ${JSON.stringify(params)}`)
+    }
+
+    callback(new Error('Url is missing'))
+    return
+  }
+
+  request.fetchPublisherInfo(params.url, {
+    method: 'GET',
+    responseType: 'text',
+    headers: {
+      'content-type': 'application/json; charset=utf-8'
+    }
+  }, (error, body) => {
+    if (error) {
+      if (params.verboseP) {
+        console.log(`roundTripFromWindow error: ${error.toString()}`)
+      }
+      return callback(error)
+    }
+
+    if (params.verboseP) {
+      console.log(`roundTripFromWindow success: ${JSON.stringify(body)}`)
+    }
+
+    return callback(null, null, body)
+  })
+}
+
+/**
+ * Round trip function for executing actions
+ * from the bat libraries (mostly server calls)
+ * @param {object} params - params that are directly tied to request.request
+ * @param {string} params.server - server url
+ * @param {string} params.method - HTTP method (GET, PUT, etc)
+ * @param {object} params.payload - payload that we want to send to the server
+ * @param {object} params.headers - HTTP headers
+ * @param {string} params.path - relative path to requested url
+ * @param {object} options
+ * @param {boolean} options.verboseP - tells us if we want to log the process or not
+ * @param {object} options.headers - headers that are used in the request.request
+ * @param {string} options.server - server url
+ * @param {boolean} options.binaryP - are we receiving binary payload back
+ * @param {boolean} options.rawP - are we receiving raw payload back
+ * @param {boolean} options.scrapeP - are we doping scraping
+ * @param {boolean} options.windowP - do we want to run this request in the window process
+ * @param {roundtripCallback} callback - The callback that handles the response
+ */
 const roundtrip = (params, options, callback) => {
   let parts = typeof params.server === 'string' ? urlParse(params.server)
     : typeof params.server !== 'undefined' ? params.server
@@ -1436,6 +1507,11 @@ const roundtrip = (params, options, callback) => {
     parts.pathname = parts.path
   }
 
+  if (options.windowP) {
+    roundTripFromWindow({url: urlFormat(parts), verboseP: options.verboseP}, callback)
+    return
+  }
+
   options = {
     url: urlFormat(parts),
     method: params.method,
@@ -2863,6 +2939,7 @@ const getMethods = () => {
       activityRoundTrip,
       pathName,
       onReferralInit,
+      roundTripFromWindow,
       onReferralCodeRead,
       onVerifiedPStatus
     }

app/extensions.js

@@ -38,7 +38,7 @@ let generateBraveManifest = () => {
     manifest_version: 2,
     version: '1.0',
     background: {
-      scripts: [ 'content/scripts/idleHandler.js' ],
+      scripts: [ 'content/scripts/metaScraper.js', 'content/scripts/requestHandler.js', 'content/scripts/idleHandler.js' ],
       persistent: true
     },
     content_scripts: [
@@ -187,16 +187,19 @@ let generateBraveManifest = () => {
     'style-src': '\'self\' \'unsafe-inline\'',
     'font-src': '\'self\' data:',
     'img-src': '* data: file://*',
+    'connect-src': 'https://www.youtube.com',
     'frame-src': '\'self\' https://brave.com'
   }
 
   if (process.env.NODE_ENV === 'development') {
     // allow access to webpack dev server resources
     let devServer = 'localhost:' + process.env.npm_package_config_port
     cspDirectives['default-src'] = '\'self\' http://' + devServer
-    cspDirectives['connect-src'] = ['\'self\'',
+    cspDirectives['connect-src'] = cspDirectives['connect-src'] + [
+      ' \'self\'',
       'http://' + devServer,
-      'ws://' + devServer].join(' ')
+      'ws://' + devServer
+    ].join(' ')
     cspDirectives['style-src'] = '\'self\' \'unsafe-inline\' http://' + devServer
     cspDirectives['font-src'] += ` http://${devServer}`
   }

app/extensions/brave/content/scripts/metaScraper.js

@@ -0,0 +1,102 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+ // Main repository
+ // Version 3.9.2
+// https://github.com/microlinkhq/metascraper
+
+// Image
+// https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-image
+
+// Author
+// https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-author
+
+// Title
+// https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-title
+
+// YouTube
+// https://github.com/microlinkhq/metascraper/tree/master/packages/metascraper-youtube
+
+const metaScraperRules = {
+  // Rules
+  getImageRules: () => {
+    const wrap = rule => ({htmlDom,url}) => {
+      const value = rule(htmlDom)
+      return requestHandlerApi.isUrl(value) && requestHandlerApi.getUrl(url, value)
+    }
+
+    return [
+      // Youtube
+      ({htmlDom,url}) => {
+        const {id,service} = requestHandlerApi.getVideoId(url)
+        return service === 'youtube' && id && requestHandlerApi.getThumbnailUrl(id)
+      },
+      // Regular
+      wrap(html => requestHandlerApi.getContent(html.querySelector('meta[property="og:image:secure_url"]'))),
+      wrap(html => requestHandlerApi.getContent(html.querySelector('meta[property="og:image:url"]'))),
+      wrap(html => requestHandlerApi.getContent(html.querySelector('meta[property="og:image"]'))),
+      wrap(html => requestHandlerApi.getContent(html.querySelector('meta[name="twitter:image:src"]'))),
+      wrap(html => requestHandlerApi.getContent(html.querySelector('meta[name="twitter:image"]'))),
+      wrap(html => requestHandlerApi.getContent(html.querySelector('meta[name="sailthru.image.thumb"]'))),
+      wrap(html => requestHandlerApi.getContent(html.querySelector('meta[name="sailthru.image.full"]'))),
+      wrap(html => requestHandlerApi.getContent(html.querySelector('meta[name="sailthru.image"]'))),
+      wrap(html => requestHandlerApi.getValue(html.querySelectorAll('article img[src]'), requestHandlerApi.getSrc)),
+      wrap(html => requestHandlerApi.getValue(html.querySelectorAll('#content img[src]'), requestHandlerApi.getSrc)),
+      wrap(html => requestHandlerApi.getSrc(html.querySelector('img[alt*="author"]'))),
+      wrap(html => requestHandlerApi.getSrc(html.querySelector('img[src]')))
+    ]
+  },
+
+  getTitleRules: () => {
+    const wrap = rule => ({htmlDom}) => {
+      const value = rule(htmlDom)
+      return requestHandlerApi.isString(value) && requestHandlerApi.titleize(value)
+    }
+
+    return [
+      // Regular
+      wrap(html => requestHandlerApi.getContent(html.querySelector('meta[property="og:title"]'))),
+      wrap(html => requestHandlerApi.getContent(html.querySelector('meta[name="twitter:title"]'))),
+      wrap(html => requestHandlerApi.getContent(html.querySelector('meta[name="sailthru.title"]'))),
+      wrap(html => requestHandlerApi.getText(html.querySelector('.post-title'))),
+      wrap(html => requestHandlerApi.getText(html.querySelector('.entry-title'))),
+      wrap(html => requestHandlerApi.getText(html.querySelector('[itemtype="http://schema.org/BlogPosting"] [itemprop="name"]'))),
+      wrap(html => requestHandlerApi.getText(html.querySelector('h1[class*="title"] a'))),
+      wrap(html => requestHandlerApi.getText(html.querySelector('h1[class*="title"]'))),
+      wrap(html => requestHandlerApi.getText(html.querySelector('title')))
+    ]
+  },
+
+  getAuthorRules: () => {
+    const wrap = rule => ({htmlDom}) => {
+      const value = rule(htmlDom)
+
+      return requestHandlerApi.isString(value) &&
+        !requestHandlerApi.isUrl(value, {relative: false}) &&
+        requestHandlerApi.titleize(value, {removeBy: true})
+    }
+
+    return [
+      // Youtube
+      wrap(html => requestHandlerApi.getText(html.querySelector('#owner-name'))),
+      wrap(html => requestHandlerApi.getText(html.querySelector('#channel-title'))),
+      wrap(html => requestHandlerApi.getValue(html.querySelectorAll('[class*="user-info"]'))),
+      // Regular
+      wrap(html => requestHandlerApi.getContent(html.querySelector('meta[property="author"]'))),
+      wrap(html => requestHandlerApi.getContent(html.querySelector('meta[property="article:author"]'))),
+      wrap(html => requestHandlerApi.getContent(html.querySelector('meta[name="author"]'))),
+      wrap(html => requestHandlerApi.getContent(html.querySelector('meta[name="sailthru.author"]'))),
+      wrap(html => requestHandlerApi.getValue(html.querySelectorAll('[rel="author"]'))),
+      wrap(html => requestHandlerApi.getValue(html.querySelectorAll('[itemprop*="author"] [itemprop="name"]'))),
+      wrap(html => requestHandlerApi.getValue(html.querySelectorAll('[itemprop*="author"]'))),
+      wrap(html => requestHandlerApi.getContent(html.querySelector('meta[property="book:author"]'))),
+      requestHandlerApi.strict(wrap(html => requestHandlerApi.getValue(html.querySelectorAll('a[class*="author"]')))),
+      requestHandlerApi.strict(wrap(html => requestHandlerApi.getValue(html.querySelectorAll('[class*="author"] a')))),
+      requestHandlerApi.strict(wrap(html => requestHandlerApi.getValue(html.querySelectorAll('a[href*="/author/"]')))),
+      wrap(html => requestHandlerApi.getValue(html.querySelectorAll('a[class*="screenname"]'))),
+      requestHandlerApi.strict(wrap(html => requestHandlerApi.getValue(html.querySelectorAll('[class*="author"]')))),
+      requestHandlerApi.strict(wrap(html => requestHandlerApi.getValue(html.querySelectorAll('[class*="byline"]'))))
+    ]
+  }
+}